@tremho/mist-lift 2.2.9 → 2.4.0

package/src/commands/publish.ts

@@ -28,13 +28,17 @@ import { addBuiltInDefinitions, MakeBuiltinApiDoc } from './builtin/ApiDocMaker'
 import { DeployApiBuiltin, DeployRootFileserves, DeployWebrootBuiltIn } from './builtin/BuiltInHandler'
 import { esbuilder } from '../lib/ESBuild'
 import { getIdSrc, getIdDelimiter } from '../lib/IdSrc'
+import { ExportWebroot, getWebrootSettings } from './builtin/ExportWebroot'
+import { doDeployAsync } from './deploy'
+import updateDeployedPermissions from './actions/updateDeployedPermissions'
 
 let projectPaths: any
 
-export async function doPublishAsync (stageName?: string): Promise<number> {
-  // let retCode = await doDeployAsync([])
-  // if(retCode) return retCode
-  const retCode = 0
+export async function doPublishAsync (args: string[] = []): Promise<number> {
+  const stageName = 'Dev'
+  const retCode = await doDeployAsync(args)
+  if (retCode) return retCode
+  // const retCode = 0
 
   projectPaths = resolvePaths()
   if (projectPaths.verified !== true) {
@@ -57,9 +61,10 @@ async function publishApi (
   await esbuilder(null, true)
   // do the built-in deploys
   // console.log(ac.gray.dim('>> webroot '))
-  await DeployWebrootBuiltIn()
-  // console.log(ac.gray.dim('>> fileserves '))
-  await DeployRootFileserves()
+  // await DeployWebrootBuiltIn()
+  // // console.log(ac.gray.dim('>> fileserves '))
+  // await DeployRootFileserves()
+  await ExportWebroot()
   // console.log(ac.gray.dim('>> api built in '))
   await DeployApiBuiltin()
   // console.log(ac.gray.dim('>> past api built in '))
@@ -131,15 +136,20 @@ async function publishApi (
   if (apiId === '') return
 
   console.log(ac.grey('Continuing with binding...'))
+  // console.log("prerequisite values")
   const prereq = await PrequisiteValues(apiId ?? '')
-  // console.log(ac.gray.dim('>> making intRequests'))
+  // console.log('>> making intRequests')
   const intRequests = prereq.MakeRequests()
-  // console.log(ac.magenta.dim('>> putting integration'), intRequests)
+  // console.log('>> putting integration')
+  // console.log(ac.green("finalizing deployment..."))
   await PutIntegrations(intRequests)
+  // console.log("deploy api")
   const success = await DeployApi(apiId ?? '', stageName)
   const region = getSettings()?.awsPreferredRegion ?? ''
   const publishUrl = `https://${apiId ?? ''}.execute-api.${region}.amazonaws.com/${stageName}`
   const characterize = success ? 'Successfully' : ''
+  console.log(ac.green.italic('Updating bound function permissions...'))
+  await updateDeployedPermissions(apiId)
   console.log(ac.green.bold(`\n ${characterize} deployed to ${publishUrl}`))
   recordLatestPublish(publishUrl)
 }
@@ -224,17 +234,19 @@ class PrereqInfo {
   }
 
   public findARN (name: string): string {
+    // console.log("findARN")
     let lastDelim = name.lastIndexOf(getIdDelimiter())
     if (lastDelim !== -1) name = name.substring(0, lastDelim)
     // console.log(ac.grey.dim.italic("binding "+name))
     // console.warn(`>>> finding name ${name} in ${this.functions.length} functions`)
     for (const f of this.functions ?? []) {
+      // console.log("inspecting function " + f.name)
       lastDelim = f.name.lastIndexOf(getIdDelimiter())
       const fname = f.name.substring(0, lastDelim)
       // console.warn('comparing name, fname, lc ', {lastus: lastDelim, fname, name})
       if (fname.toLowerCase() === name.toLowerCase()) {
         // console.warn(">>> Match! ", f)
-        console.log(ac.grey.dim.italic('binding ' + name))
+        // console.log(ac.grey.dim.italic('binding ' + name + ' to arn '+ f.arn))
         return f.arn
       }
     }
@@ -247,8 +259,10 @@ class PrereqInfo {
     const region = getSettings()?.awsPreferredRegion ?? ''
     const out: PutIntegrationRequest[] = []
 
+    // console.log("MakeRequests ", {region, defs:this.defs})
     for (const d of this.defs) {
       const def = (d)
+      // if(def.name === 'Webroot' || def.name.substring(0,9) === 'fileServe') continue
       // console.log(ac.magenta.dim('>> finding api and arn for '), {pathMap:def.pathMap, name: def.name})
       const api = this.findApi(def.pathMap, def.method)
       const arn = this.findARN(def.name) ?? ''
@@ -269,17 +283,27 @@ class PrereqInfo {
         })
       }
     }
+    // console.log("MakeRequests return ", {out})
     return out
   }
 }
 
 async function PrequisiteValues (id: string): Promise<PrereqInfo> {
+  // console.log("PrerequisiteValues", {id})
   const pri = new PrereqInfo()
   const out = await GetFunctionInfo(pri)
+  // console.log("function info", {functionList: out.functions})
   out.defs = gatherFunctionDefinitions()
-  addBuiltInDefinitions(out.defs)
+  // console.log("gatherFunctionDefinitions returned ", {defs: out.defs})
+  const wrs = await getWebrootSettings()
+  const withWebroot = (wrs.webrootMethod ?? 'SELF') === 'SELF'
+  // console.log("wrs.webrootMethod=",wrs.webrootMethod)
+  // console.log("addBuiltInDefinitions", {withWebroot})
+  addBuiltInDefinitions(out.defs, withWebroot)
   out.requestApiId = id
+  // console.log("getMethodResources")
   await GetMethodResources(id, out)
+  // console.log("Prerequisite Values out", JSON.stringify(out,null,2))
 
   return out
 }
@@ -324,7 +348,7 @@ async function GetMethodResources (
     restApiId: id
   })
   const response: any = await client.send(listCommand)
-  // ClogInfo("Resources response", response)
+  // console.log("Resources response", response)
   if ((response?.items) != null) info.apis = response.items
 
   return info
@@ -348,6 +372,7 @@ async function PutIntegrations (integrations: PutIntegrationRequest[]): Promise<
 async function DeployApi (restApiId: string, stageName: string): Promise<boolean> {
   let success = true
   try {
+    // console.log("Creating deployment for "+restApiId+", stage "+stageName)
     const client = new APIGatewayClient(getAWSCredentials())
     const command: any = new CreateDeploymentCommand({
       restApiId,

package/src/commands/start.ts

@@ -39,8 +39,9 @@ export async function startLocalServer (): Promise<void> {
   await functionBinder()
   // Log.Trace('done calling functionBinder')
   // for JSON posts
+  // app.use(express.raw({limit: '6mb'}))
   app.use(bodyParser.json({ limit: '50mb' }))
-  app.use(express.json())
+  app.use(express.json({ limit: '6mb' }))
   // for form posts
   app.use(express.urlencoded({ extended: true }))
 

package/src/commands/update.ts

@@ -22,6 +22,7 @@ export async function doUpdateAsync (
   try {
     // console.warn(">>> esbuilder")
     await esbuilder(null, true)
+    console.log('>>>> TODO: Call ExportWebroot on update instead of these next two functions')
     // console.warn(">>> DeployRootFileserves")
     await DeployRootFileserves()
     // console.warn(">>> DeployWebrootBuiltIn")

package/src/expressRoutes/all.ts

@@ -7,6 +7,7 @@ import { gatherFunctionDefinitions } from '../lib/openAPI/ApiBuildCollector'
 
 import express from 'express'
 const router = express.Router()
+// router.use( express.raw({type: 'application/octet-stream'}))
 
 export function allBinder (): void {
   const projectPaths = resolvePaths()

package/src/expressRoutes/functionBinder.ts

@@ -7,10 +7,12 @@ import { resolvePaths } from '../lib/pathResolve'
 import express from 'express'
 import { gatherFunctionDefinitions } from '../lib/openAPI/ApiBuildCollector'
 import { buildOpenApi } from '../lib/openAPI/openApiConstruction'
-import { Log } from '@tremho/inverse-y'
+import { BadRequest, Log } from '@tremho/inverse-y'
+import { parseConstraints } from '../lib/TypeCheck'
 
 const clearModule = require('clear-module')
 const router = express.Router()
+router.use(express.raw({ type: '*/*', limit: '6mb' }))
 
 export function functionBinder (): void {
   const defs = gatherFunctionDefinitions()
@@ -43,23 +45,38 @@ export function functionBinder (): void {
         const n = entryRoot.indexOf('/{')
         if (n !== -1) entryRoot = entryRoot.substring(0, n) + '/*'
 
-        const callHandler = (pathMap: string, req: any, res: any): void => {
-          const event = requestToEvent(pathMap, req)
+        const callHandler = (def: any, req: any, res: any): void => {
+          console.log('>>>>>>> YES WE ARE HERE<<<<<<<<<')
+          if (Buffer.isBuffer(req.body)) {
+            const buffer: Buffer = req.body
+            console.log('body is a buffer at callHandler. Length is ' + buffer.byteLength)
+            console.log('First 16 bytes:', buffer.subarray(0, 16).toString('hex'))
+          }
+          const message = testPathRoute(def, req)
+          if (message) {
+            return handleResponse(res, { statusCode: 403, body: { error: message } })
+          }
+          const event = requestToEvent(def, req)
+          try {
+            performParameterTesting(def, event)
+          } catch (e: any) {
+            return handleResponse(res, { statusCode: 400, body: { error: e.message } })
+          }
           Promise.resolve(start(event, null, null)).then(respOut => {
             handleResponse(res, respOut)
           }).catch<any>((reason: any) => undefined)
         }
 
         if (method === 'get') {
-          router.get(entryRoot, (req, res) => callHandler(pathMap, req, res))
+          router.get(entryRoot, (req, res) => callHandler(def, req, res))
         } else if (method === 'post') {
-          router.post(entryRoot, (req, res) => callHandler(pathMap, req, res))
+          router.post(entryRoot, (req, res) => callHandler(def, req, res))
         } else if (method === 'put') {
-          router.put(entryRoot, (req, res) => callHandler(pathMap, req, res))
+          router.put(entryRoot, (req, res) => callHandler(def, req, res))
         } else if (method === 'patch') {
-          router.patch(entryRoot, (req, res) => callHandler(pathMap, req, res))
+          router.patch(entryRoot, (req, res) => callHandler(def, req, res))
         } else if (method === 'delete') {
-          router.delete(entryRoot, (req, res) => callHandler(pathMap, req, res))
+          router.delete(entryRoot, (req, res) => callHandler(def, req, res))
         } else {
           console.log(ac.red.bold('Cannot map method ') + ac.blue.bold(method))
         }
@@ -73,19 +90,136 @@ export function functionBinder (): void {
   }).catch<any>((reason: any) => undefined)
 }
 
-function requestToEvent (template: string, req: any): any {
-  // console.log('requestToEvent (express)...', req.body, typeof (req.body))
+// compare request route to the definition of this function to see if it matches
+function testPathRoute (def: any, req: any) {
+  let path = req.path
+  let pathMap = def.pathMap
+  if (path.endsWith('/')) path = path.substring(0, path.length - 1)
+  if (pathMap.endsWith('/')) pathMap = pathMap.substring(0, path.length - 1)
+  const defParts = pathMap.split('/')
+  const pathParts = path.split('/')
+
+  let message = ''
+  if (pathParts.length > defParts.length) {
+    message = 'Too many parameters passed on path'
+  }
+  if (pathParts.length < defParts.length) {
+    message = 'Too few parameters passed on path'
+  }
+  return message
+}
+
+function performParameterTesting (def: any, event: any) {
+  // for every parameter defined in def
+  // find the corresponding value in event.parameters (by name)
+  // find the constraints for that parameter in def
+  // test the value against the constraints
+  for (const pdef of (def.parameters ?? [])) {
+    const constraints = Array.isArray(pdef.constraints) ? parseConstraints(pdef.type, pdef.constraints.join('\n'), '\n') : undefined
+    let value: any = event.parameters[pdef.name]
+    if (value === undefined) {
+      if (!pdef.required) return // okay to be undefined
+    }
+    if (pdef.type === 'number') value = Number(value) // TODO: Type coercion for all types
+    constraints?.test(value)
+  }
+}
+
+// returns 'text', 'json', or undefined
+function bodyTypeCollapse (type: string) {
+  const options: any = {
+    text: 'text',
+    json: 'json',
+    'application/json': 'json'
+  }
+  if (type) {
+    type = type.toLowerCase().trim()
+    if (type.substring(0, 5) === 'text/') return 'text'
+    return options[type]
+  }
+}
+
+function requestToEvent (def: any, req: any): any {
+  console.warn('requestToEvent (express)... body is type ', typeof (req.body))
+  const isBuffer = Buffer.isBuffer(req.body)
+  console.log('Body is a buffer ', isBuffer)
+  console.log('bodyType from def is ', def.bodyType)
+  // lets assume template says the type
+  const type = bodyTypeCollapse(def.bodyType)
+  console.log('collapsed bodyType is ', type)
+  if (type === 'text') {
+    console.log('>> we want plain text')
+    if (typeof req.body === 'object') {
+      console.log('>> requestToEvent modifying body (text)')
+      if (Buffer.isBuffer(req.body)) {
+        req.body = req.body.toString()
+        console.log('>> just turned it to a string ', req.body)
+      } else {
+        req.body = JSON.stringify(req.body)
+        console.log('>> is is an object, make it a json string ', req.body)
+      }
+    }
+  } else if (type === 'json') {
+    console.log('>> we want json')
+    if (typeof req.body === 'object') {
+      if (Buffer.isBuffer(req.body)) {
+        console.log('>> converting first from buffer to string')
+        req.body = req.body.toString()
+      }
+    }
+    if (typeof req.body === 'string') {
+      console.log('>> body is a string', req.body)
+      try {
+        console.log('>> requestToEvent modifying body (json2)')
+        req.body = JSON.parse(req.body)
+        console.log(">> it's now an object ", req.body)
+      } catch (e: any) {
+        console.error(ac.bold.red('Failed request body JSON parse '), req.body)
+      }
+    } else {
+      console.log('We already have an object here as ', req.body)
+    }
+  } else {
+    console.log('>> binary type path')
+    // binary expects a buffer
+    if (!Buffer.isBuffer(req.body)) { // if already a buffer, leave as is
+      if (typeof req.body === 'object') {
+        req.body = JSON.stringify(req.body) // make a string first before we bufferize the json
+      }
+      req.body = Buffer.from(req.body)
+    }
+  }
+  if (Buffer.isBuffer(req.body)) {
+    const buffer: Buffer = req.body
+    console.log('body is buffer eventToRequest, Length = ' + buffer.byteLength)
+    console.log('First 16 bytes:', buffer.subarray(0, 16).toString('hex'))
+  }
+  const template = def.pathMap
+
   // unpack weird choice of express when passed from curl.
-  if (typeof (req.body) === 'object') {
-    const bprops = Object.getOwnPropertyNames(req.body)
+  /*
+  if (typeof (req.body) === 'object' && !Buffer.isBuffer(req.body)) {
+    console.log(">> in curl case", req.body)
+    let bprops:string[] = []
+    try {
+      bprops = Object.getOwnPropertyNames(req.body)
+      console.log(">> bprops ", bprops)
+    } catch(e:any) {}
     if (bprops.length === 1) {
+      const bText = req.body[bprops[0]]
+      console.log(">> bText ",bText)
       try {
-        req.body = JSON.parse(bprops[0])
+        req.body = JSON.parse(bText)
       } catch (e: any) {
-        console.error('failed request body parse', bprops[0])
+        console.warn(ac.bold.yellow.italic('Failed request body JSON parse Yellow'), '(curl)')
+        req.body = bText
+        return requestToEvent(def, req)
       }
     }
   }
+   */
+
+  console.log('last look at body', req.body)
 
   let path: string = req.originalUrl ?? ''
   const qi = path.indexOf('?')
@@ -115,7 +249,9 @@ function requestToEvent (template: string, req: any): any {
     const brknm = (tslots[i] ?? '').trim()
     if (brknm.charAt(0) === '{') {
       const pn = brknm.substring(1, brknm.length - 1)
-      parameters[pn] = (pslots[i] ?? '').trim()
+      let pv: string | undefined = (pslots[i] ?? '').trim()
+      if (pv === 'undefined' || pv === '~') pv = undefined
+      parameters[pn] = pv
     }
   }
   for (const p of Object.getOwnPropertyNames(req.query)) {
@@ -125,7 +261,7 @@ function requestToEvent (template: string, req: any): any {
   // console.warn(">>> functionBinder geting headers", {req, headers})
   // const apiKey = req.header('x-api-key')
   // console.warn(">>> apiKey returned using header function", {apiKey})
-  console.log('making eventOut with req.body=', req.body)
+  // console.log('making eventOut with req.body=', req.body)
   const eventOut: any = {
     requestContext: req,
     request: {

package/src/lib/DirectoryUtils.ts

@@ -7,7 +7,8 @@ export type RecurseCB = (filepath: string, stats: Stats) => boolean
 
 // Recurse a directory, calling back to a 'for-each' callback for all files in tree
 export function recurseDirectory (dirpath: string, callback?: RecurseCB): void {
-  fs.readdirSync(dirpath).forEach((file: string) => {
+  const fl = fs.readdirSync(dirpath)
+  fl.forEach((file: string) => {
     const fpath = path.join(dirpath, file)
     // if(fpath.indexOf('node_modules') === -1) {
     const stat = fs.statSync(fpath)

package/src/lib/IdSrc.ts

@@ -1,6 +1,7 @@
 
 import { getProjectName, getProjectVersion } from '../lib/LiftVersion'
 import sha1 from 'sha1'
+import { STSClient, GetCallerIdentityCommand } from '@aws-sdk/client-sts'
 
 export function getIdDelimiter (): string {
   return '_-_'
@@ -10,10 +11,14 @@ export function getIdSrc (): string {
   while (idsrc.includes(' ')) idsrc = idsrc.replace(' ', '_')
   while (idsrc.includes('.')) idsrc = idsrc.replace('.', '-')
 
+  const projectHash = sha1(idsrc).toString().substring(0, 8)
+  // console.log("project hash", {idsrc, projectHash})
+  idsrc = getProjectName() + '_' + projectHash
+  // console.log("idsrc is "+idsrc)
   return idsrc
 }
 export function decoratedName (name: string): string {
-  if (name.includes(getIdDelimiter())) {
+  if (name.includes(getIdDelimiter()) && name.includes(getIdSrc())) {
     // console.warn('name '+name+' appears to be decorated already')
     return name
   }
@@ -22,13 +27,21 @@ export function decoratedName (name: string): string {
 
   const hash8 = sha1(name.toLowerCase()).toString().substring(0, 8)
 
-  const fullName = name + getIdDelimiter() + getIdSrc()
+  const fullName = name + getIdDelimiter() + '_' + getIdSrc()
+  // console.log("fullname is "+fullName)
   let dname
   if (name.startsWith('fileserve_')) {
-    dname = ('fileServe_' + hash8 + getIdDelimiter() + getIdSrc()).substring(0, 64)
+    dname = ('fileServe_' + hash8 + getIdDelimiter() + '_' + getIdSrc()).substring(0, 64)
   } else {
-    dname = fullName.substring(0, 50) + '_' + hash8
+    dname = fullName // .substring(0, 50) + '_' + hash8
   }
   // console.warn('decorated to '+dname+' from '+name)
   return dname
 }
+
+export async function getAccountId () {
+  const client = new STSClient({} as any)
+  const command: any = new GetCallerIdentityCommand({})
+  const response: any = await client.send(command)
+  return response.Account // This is your AWS account ID
+}