@architect/inventory 3.0.0-RC.1 → 3.0.0-RC.10

package/changelog.md

@@ -6,14 +6,23 @@
 
 ### Added
 
+- Architect 10 plugin API support! Specifically:
+  - `plugins.set.runtimes` - custom runtime support (still in beta)
+  - `plugins.set.env` - add environment variables to all Lambdas
+  - `plugins.set.events|http|scheduled|tables-streams|ws` - generate or drop in Lambdas in Architect pragmas
+  - `plugins.set.customLambdas` - generate or drop in unique Lambdas with custom event sources
+  - More below...
 - Added `inv|get.plugins` tree + methods
   - What used to be `plugins` in the plugins beta is now `customLambdas` (see next item)
 - Added `inv|get.customLambdas`
   - Formerly `inv|get.plugins`
 - Added `inv._project.customRuntimes`
-- Added the concept of a `build` destination (currently only available via plugin)
+- Added low-level support for `build` destinations to runtime plugins that register type `transpiled` or `compiled`
 - Added `handlerModuleSystem` property for `nodejs14.x` Lambdas, with a value of `cjs` or `esm` based on Lambda + Node.js conventions
 - Added `handlerFile` detection for `nodejs14.x` + `deno` Lambdas
+  - This will detect the correct handler file on the filesystem, and fall back to a default handler file if none are found (e.g. `index.js` in `nodejs14.x`)
+- Added `inv._arc.deployStage` property, enabling Inventory to be aware of an intended deploy stage; (this property may change, consider it in beta!)
+- Added built-in support for reading `.env` files when enumerating local env var preferences
 
 
 ### Changed
@@ -21,15 +30,27 @@
 - Breaking change: changed `_project.src`, added `_project.cwd`, making both the pair significantly more literal and descriptive
   - `_project.src` is now the default source tree folder (eg `$cwd/src`)
   - `_project.cwd` refers to the current working directory of the project
+- Breaking change: `_project.env` is now by default an object populated by three properties: `local`, `plugins`, and `aws`, reflecting the env vars found for each environment
 - Breaking change: AWS region prioritizes a region passed via param over `AWS_REGION` env var; this should realistically have little or no effect in practice
 - Breaking change: legacy `@tables-streams` folders (`src/tables/...` and `src/streams/...`) are now deprecated
   - Existing functions can be simply moved to `src/tables-streams/{name}` (or use a custom `src` property)
 - Breaking change: renamed `lambda.handlerFunction` to `lambda.handlerMethod`
-- Breaking change: prioritize mod.ts|js in Deno
-- Internal change: performance improvements to building `inv.shared` + `inv.views`
+- Breaking change: prioritize `mod.ts|js` handlers in Deno Lambdas
+- Breaking change: removed `toml` support
+- Breaking change: removed `get.macros` method; as `@macros` are now automatically mapped to the Architect plugins, you can simply use `get.plugins` instead
+- Performance improvements to building `inv.shared` + `inv.views`
 - Improved memory footprint of Inventory object by preserving references in `lambdaSrcDirs`, `lambdasBySrcDir`
-  - Added `pragma` property to all Lambdas to preserve references
+  - Added `pragma` property to all Lambdas to aid in reference preservation
 - Tidy up order of enumerated properties in each Lambda
+Update CI
+- Stop publishing to the GitHub Package registry
+
+
+### Fixed
+
+- Added file path validation because `aws-sdk` blows up on !ascii paths; fixes #1292, thanks @GustMartins!
+- Fixed env var validation from preference files
+- Fixed error bubbling when reading a preferences file with issues
 
 ---
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@architect/inventory",
-  "version": "3.0.0-RC.1",
+  "version": "3.0.0-RC.10",
   "description": "Architect project resource enumeration utility",
   "main": "src/index.js",
   "scripts": {
@@ -9,7 +9,8 @@
     "test:integration": "cross-env tape 'test/integration/**/*-test.js' | tap-spec",
     "coverage": "nyc --reporter=lcov --reporter=text npm run test:unit",
     "lint": "eslint . --fix",
-    "rc": "npm version prerelease --preid RC"
+    "rc": "npm version prerelease --preid RC",
+    "vendor": "scripts/vendor"
   },
   "engines": {
     "node": ">=14"
@@ -20,27 +21,28 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
-    "@architect/asap": "~4.1.0",
-    "@architect/parser": "~5.0.2",
-    "@architect/utils": "~3.0.4",
+    "@architect/asap": "~5.0.0-RC.0",
+    "@architect/parser": "~6.0.0-RC.0",
+    "@architect/utils": "~3.1.0-RC.0",
     "lambda-runtimes": "~1.1.1"
   },
   "devDependencies": {
     "@architect/eslint-config": "~2.0.1",
-    "aws-sdk": "2.880.0",
-    "aws-sdk-mock": "~5.5.0",
+    "aws-sdk": "2.1001.0",
+    "aws-sdk-mock": "~5.6.2",
     "cross-env": "~7.0.3",
-    "eslint": "~8.5.0",
+    "dotenv": "~15.0.0",
+    "eslint": "~8.8.0",
     "mock-fs": "~5.1.2",
     "mock-require": "~3.0.3",
     "nyc": "~15.1.0",
     "tap-spec": "^5.0.0",
-    "tape": "^5.3.2"
+    "tape": "^5.5.0"
   },
   "eslintConfig": {
     "extends": "@architect/eslint-config"
   },
-  "_nyc": {
+  "nyc": {
     "check-coverage": true,
     "branches": 100,
     "lines": 100,

package/readme.md

@@ -43,7 +43,7 @@ The inventory object contains the entirety of a project's data, including Archit
 
 Top-level inventory parameters that start with an underscore (e.g. `_arc`, `_project`) denote project metadata or internal diagnostic data; all other parameters represent userland project resources.
 
-In a project inventory, `null` values are used as placeholders for known values or options that were not user-defined. The existence of a non-`null` value can be inferred as a user having specifically defined a setting. For example: `arc.http === null` can be construed as the user having **not** defined an `@http` pragma. This rule has some exceptions:
+In a project inventory, `null` values are used as placeholders for known values or options that were not user-defined. The existence of a non-`null` value can be inferred as a user having specifically defined a setting. For example: `arc.http: null` can be construed as the user having **not** defined an `@http` pragma. This rule has some exceptions:
 
 - A handful of settings that must be backfilled if not supplied
   - Example: `inv.aws.region`, which is required by the `aws-sdk` to function, and will be backfilled if not defined
@@ -51,9 +51,10 @@ In a project inventory, `null` values are used as placeholders for known values
   - Example: while `@static` can be defined on its own without any other pragmas, the existence of `@http` infers `@static`
   - Thus, the act of adding `@http` will necessarily make `inv.static` non-`null`
 - Settings that generate related resources
-  - Example: DynamoDB streams can be defined in `@tables` with `stream true`; Inventory would interpret a table with `stream true` as a new `inv.streams` resource and thus make `inv.streams` non-`null`
-
-> Note: The `inv` format is primarily designed and intended for internal use within Architect libraries; as such, Inventory may theoretically introduce breaking changes in its behavior, shape, or naming conventions.
+  - Example: DynamoDB streams can be defined in `@tables` with `stream true`; Inventory would interpret a table with `stream true` as a new `inv['tables-streams']` resource and thus make `inv['tables-streams']` non-`null`
+- Lambda `handlerFile` file path property is present even if the file is not
+  - This differs from Lambda `configFile` file path properties, which will be `null` if no file is present
+  - This exception is namely because some workflows may need the computed default handler path (example: when running `arc create`)
 
 
 #### `get`

package/src/config/arc.js

@@ -6,14 +6,14 @@ let { join } = require('path')
  */
 module.exports = function getArcConfig (params) {
   let { cwd, inventory } = params
-  let arc = { ...inventory._arc }
+  let _arc = { ...inventory._arc }
 
   // Version
   let installed = join(cwd, 'node_modules', '@architect', 'architect', 'package.json')
   if (existsSync(installed)) {
     let { version } = JSON.parse(readFileSync(installed))
-    arc.version = version
+    _arc.version = version
   }
 
-  return arc
+  return _arc
 }

package/src/config/pragmas/plugins.js

@@ -1,10 +1,11 @@
 let { join } = require('path')
 let { existsSync } = require('fs')
-let { is, normalizeSrc, pragmas } = require('../../lib')
+let { is, normalizeSrc, pragmas, tidyError, validationPatterns } = require('../../lib')
 let { lambdas } = pragmas
 let nonLambdaSetters = [ 'customLambdas', 'env', 'runtimes' ]
 let setters = [ ...lambdas, ...nonLambdaSetters ]
 let pluginMethods = [ 'deploy', 'sandbox' ] // TODO add more!
+let reservedNames = [ '_methods', 'events', 'queues', 'static', 'tables' ]
 
 module.exports = function getPluginModules ({ arc, inventory, errors }) {
   if (!arc?.plugins?.length && !arc?.macros?.length) return null
@@ -33,8 +34,12 @@ module.exports = function getPluginModules ({ arc, inventory, errors }) {
         : join(cwd, 'src', type + 's', name)
     }
 
-    if (name === '_methods') {
-      errors.push('Plugin name _methods is reserved, please rename your plugin')
+    if (reservedNames.includes(name)) {
+      errors.push(`Plugin name ${name} is reserved, please rename your plugin`)
+      continue
+    }
+    if (!validationPatterns.veryLooseName.test(name)) {
+      errors.push('Plugin names can only contain [a-zA-Z0-9/\\-._]')
       continue
     }
     if (pluginPath) {
@@ -86,7 +91,7 @@ module.exports = function getPluginModules ({ arc, inventory, errors }) {
         })
       }
       catch (err) {
-        errors.push(`Unable to load plugin '${name}': ${err.message.split('\n')[0]}`)
+        errors.push(`Unable to load plugin '${name}': ${tidyError(err)}`)
       }
     }
     else errors.push(`Cannot find plugin '${name || plugin}'! Are you sure you have installed or created it correctly?`)

package/src/config/pragmas/populate-lambda/index.js

@@ -1,3 +1,5 @@
+let { sep } = require('path')
+let { deepFrozenCopy } = require('@architect/utils')
 let read = require('../../../read')
 let getLambda = require('./get-lambda')
 let getRuntime = require('./get-runtime')
@@ -9,22 +11,45 @@ let { compiledRuntimes, is } = require('../../../lib')
 /**
  * Build out the Lambda tree from the Arc manifest or a passed pragma, and plugins
  */
-function populateLambda (type, { arc, inventory, errors, pragma }) {
+function populateLambda (type, params) {
+  // Passing a pragma array via params allows special overrides
+  // See: @tables populating inv['tables-streams']
+  let { arc, inventory, errors, pragma } = params
+
   let plugins = inventory.plugins?._methods?.set?.[type]
   let pluginLambda = []
   if (plugins) {
+    let invCopy = deepFrozenCopy(inventory)
     let pluginResults = plugins.flatMap(fn => {
-      let result = fn({ arc, inventory })
-      if (!result) {
+      try {
+        var result = fn({ arc: invCopy._project.arc, inventory: { inv: invCopy } })
+      }
+      catch (err) {
+        err.message = `Setter plugin exception: plugin: ${fn.plugin}, method: set.${type}`
+                      + `\n` + err.message
+        throw err
+      }
+      if (!result ||
+          (!is.object(result) && !is.array(result)) ||
+          (is.array(result) && result.some(r => !is.object(r)))) {
         errors.push(`Setter plugins must return a valid response: plugin: ${fn.plugin}, method: set.${type}`)
         return []
       }
-      result.plugin = fn.plugin
-      result.type = fn.type
+      if (is.array(result)) {
+        result.forEach(item => {
+          item.plugin = fn.plugin
+          item.type = fn.type
+        })
+      }
+      else {
+        result.plugin = fn.plugin
+        result.type = fn.type
+      }
       return result
     })
     pluginLambda = populate(type, pluginResults, inventory, errors, true) || []
   }
+
   let pragmaLambda = populate(type, pragma || arc[type], inventory, errors) || []
   let aggregate = [ ...pluginLambda, ...pragmaLambda ]
   return aggregate.length ? aggregate : null
@@ -46,6 +71,11 @@ function populate (type, pragma, inventory, errors, plugin) {
     if (!result) continue
 
     let { name, src, build } = result
+
+    // Normalize paths, especially since plugin authors may not use path.join
+    src = normalize(src)
+    if (build) build = normalize(build)
+
     // Set up fresh config, then overlay plugin config
     let config = defaultProjectConfig()
     config = { ...config, ...getKnownProps(configProps, result.config) }
@@ -82,7 +112,7 @@ function populate (type, pragma, inventory, errors, plugin) {
 
     // Tidy up any irrelevant properties
     if (!compiledRuntimes.includes(config.runtimeConfig?.type)) {
-      // Important: if we don't clean up the build prop, many explosions will explode
+      // Super important! If we don't clean up the build prop, many explosions will explode
       build = undefined
     }
     if (type !== 'http') {
@@ -111,8 +141,10 @@ function populate (type, pragma, inventory, errors, plugin) {
   return lambdas
 }
 
+let normalize = path => path.replace(/[\\\/]/g, sep)
+
 // Lambda setter plugins can technically return anything, so this ensures everything is tidy
-let lambdaProps = [ 'cron', 'method', 'path', 'plugin', 'rate', 'route', 'table' ]
+let lambdaProps = [ 'cron', 'method', 'path', 'plugin', 'rate', 'route', 'table', 'type' ]
 let configProps = [ ...Object.keys(defaultFunctionConfig()), 'fifo', 'views' ]
 let getKnownProps = (knownProps, raw = {}) => {
   let props = knownProps.flatMap(prop => is.defined(raw[prop]) ? [ [ prop, raw[prop] ] ] : [])

package/src/config/pragmas/static.js

@@ -1,8 +1,10 @@
 let { asapSrc, is } = require('../../lib')
 
 module.exports = function configureStatic ({ arc, inventory }) {
+  let httpSetters = inventory.plugins?._methods?.set?.http
+
   // @static is inferred by @http
-  if (!arc.static && !arc.http) return null
+  if (!arc.static && !arc.http && !httpSetters) return null
 
   let staticPragma = arc.static || []
   let _static = {
@@ -37,7 +39,7 @@ module.exports = function configureStatic ({ arc, inventory }) {
   }
 
   // Handy shortcut to ASAP for bare @static
-  if (!arc.http) {
+  if (!arc.http && !httpSetters) {
     inventory._project.rootHandler = 'arcStaticAssetProxy'
     inventory._project.asapSrc = asapSrc()
   }

package/src/config/pragmas/validate/_lib.js

@@ -1,15 +1,8 @@
-let { is } = require('../../../lib')
-
-let patterns = {
-  looseName: new RegExp(/^[a-z][a-zA-Z0-9-_]+$/),
-  strictName: new RegExp(/^[a-z][a-z0-9-]+$/),
-  // DynamoDB, SNS, SQS
-  veryLooseName: new RegExp(/^[a-zA-Z0-9/\-._]*$/),
-}
+let { is, validationPatterns } = require('../../../lib')
 
 function regex (value, pattern, pragmaName, errors) {
-  if (!patterns[pattern]) throw ReferenceError(`Invalid validation pattern specified: ${pattern}`)
-  if (!patterns[pattern].test(value)) errors.push(`Invalid ${pragmaName} item: '${value}' must match ${patterns[pattern]}`)
+  if (!validationPatterns[pattern]) throw ReferenceError(`Invalid validation pattern specified: ${pattern}`)
+  if (!validationPatterns[pattern].test(value)) errors.push(`Invalid ${pragmaName} item: '${value}' must match ${validationPatterns[pattern]}`)
 }
 
 function size (value, min, max, pragmaName, errors) {

package/src/config/pragmas/views.js

@@ -48,8 +48,8 @@ module.exports = function configureViews ({ arc, pragmas, inventory, errors }) {
 
     // Set new views settings
     for (let view of arc.views) {
-      let method = view[0].toLowerCase()
-      let path = view[1]
+      let method = view?.[0]?.toLowerCase()
+      let path = view?.[1]
       if (method === 'src') continue
       let name = `${method} ${path}`
       let route = pragmas.http.find(n => n.name === name)

package/src/config/project/index.js

@@ -2,25 +2,25 @@ let { join } = require('path')
 let upsert = require('../_upsert')
 let prefs = require('./prefs')
 let plugins = require('./plugins')
-let { is } = require('../../lib')
+let { is, mergeEnvVars } = require('../../lib')
 
 /**
  * Get the project-level configuration, overlaying arc.aws settings (if present)
  */
 module.exports = function getProjectConfig (params) {
   let { arc, errors, raw, filepath, inventory } = params
-  let project = {
+  let _project = {
     ...inventory._project,
     arc,
     raw,
   }
 
   if (arc.aws) {
-    project.defaultFunctionConfig = upsert(project.defaultFunctionConfig, arc.aws)
+    _project.defaultFunctionConfig = upsert(_project.defaultFunctionConfig, arc.aws)
   }
 
   if (filepath) {
-    project.manifest = filepath
+    _project.manifest = filepath
   }
 
   // Parse local and global project preferences
@@ -29,37 +29,48 @@ module.exports = function getProjectConfig (params) {
     let p = prefs({ scope, inventory, errors })
     if (p) {
       // Set up the scoped metadata
-      project[`${scope}Preferences`] = p.preferences
-      project[`${scope}PreferencesFile`] = p.preferencesFile
+      _project[`${scope}Preferences`] = p.preferences
+      _project[`${scope}PreferencesFile`] = p.preferencesFile
 
       // Build out the final preferences
       /* istanbul ignore else: jic */
-      if (!project.preferences) project.preferences = {}
+      if (!_project.preferences) _project.preferences = {}
       Object.keys(p.preferences).forEach(pragma => {
         // Ignore the raw data
         if (pragma === '_arc' || pragma === '_raw') return
         // Allow booleans, etc.
         if (!is.object(p.preferences[pragma])) {
-          project.preferences[pragma] = p.preferences[pragma]
+          _project.preferences[pragma] = p.preferences[pragma]
           return
         }
         // Traverse and merge individual settings
         /* istanbul ignore else: jic */
-        if (!project.preferences[pragma]) project.preferences[pragma] = {}
+        if (!_project.preferences[pragma]) _project.preferences[pragma] = {}
         Object.entries(p.preferences[pragma]).forEach(([ setting, value ]) => {
-          project.preferences[pragma][setting] = value
+          _project.preferences[pragma][setting] = value
         })
       })
     }
   }
 
+  // Populate local env from preferences
+  if (_project.preferences?.env) {
+    _project.env.local = _project.preferences.env
+  }
+
   if (inventory.plugins?._methods) {
-    // TODO: project.env = plugins.env(params, project)
+    _project.env.plugins = plugins.env(params, _project)
+    _project.env.local = mergeEnvVars({
+      name: 'Local',
+      source: _project.env.plugins,
+      target: _project.env.local,
+      errors,
+    })
 
-    let { build, runtimes } = plugins.runtimes(params, project)
-    if (build)    project.build = join(project.cwd, build)
-    if (runtimes) project.customRuntimes = runtimes
+    let { build, runtimes } = plugins.runtimes(params, _project)
+    if (build) _project.build = join(_project.cwd, build)
+    if (runtimes) _project.customRuntimes = runtimes
   }
 
-  return project
+  return _project
 }

package/src/config/project/plugins/env.js

@@ -0,0 +1,71 @@
+let { deepFrozenCopy } = require('@architect/utils')
+let { is, tidyError, validationPatterns: valid } = require('../../../lib')
+let envs = [ 'testing', 'staging', 'production' ]
+let str = value => {
+  if (is.object(value) || is.array(value)) return JSON.stringify(value)
+  return String(value)
+}
+
+module.exports = function setEnvPlugins (params, project) {
+  let { errors, inventory } = params
+  let envPlugins = inventory.plugins?._methods?.set?.env
+  if (envPlugins?.length) {
+    let env = {
+      testing: null,
+      staging: null,
+      production: null,
+    }
+
+    // inventory._project is not yet built, so provide as much as we can to plugins for now
+    let inv = deepFrozenCopy({ ...inventory, _project: project })
+    envPlugins.forEach(fn => {
+      let errType = `plugin: ${fn.plugin}, method: set.env`
+      try {
+        let result = fn({ arc: inv._project.arc, inventory: { inv } })
+        if (!is.object(result) || !Object.keys(result).length) {
+          return errors.push(`Env plugin returned invalid data, must return an Object with one or more keys + values: ${errType}`)
+        }
+        // Populate env vars based on environment
+        // If any keys are environment names, disregard all keys except environment names
+        if (Object.keys(result).some(k => envs.includes(k))) {
+          envs.forEach(e => {
+            if (result[e]) Object.entries(result[e]).forEach(([ k, v ]) => {
+              if (!valid.envVar.test(k)) {
+                return errors.push(`Env var '${k}' is invalid, must be [a-zA-Z0-9_]`)
+              }
+              let errored = false, val = str(v)
+              if (!env[e]) env[e] = { [k]: val }
+              else if (env[e][k] && !errored) {
+                errored = true
+                errors.push(`Env var '${k}' already registered: ${errType}`)
+              }
+              else env[e][k] = val
+            })
+          })
+        }
+        // Populate all environments based on env var
+        else {
+          Object.entries(result).forEach(([ k, v ]) => {
+            if (!valid.envVar.test(k)) {
+              return errors.push(`Env var '${k}' is invalid, must be [a-zA-Z0-9_]`)
+            }
+            let errored = false, val = str(v)
+            envs.forEach(e => {
+              if (!env[e]) env[e] = { [k]: val }
+              else if (env[e][k] && !errored) {
+                errored = true
+                errors.push(`Env var '${k}' already registered: ${errType}`)
+              }
+              else env[e][k] = val
+            })
+          })
+        }
+      }
+      catch (err) {
+        errors.push(`Runtime plugin '${fn.plugin}' failed: ${tidyError(err)}`)
+      }
+    })
+    return env
+  }
+  return inventory._project.env.plugins
+}

package/src/config/project/plugins/index.js

@@ -1,5 +1,7 @@
+let env = require('./env')
 let runtimes = require('./runtimes')
 
 module.exports = {
+  env,
   runtimes,
 }

package/src/config/project/plugins/runtimes.js

@@ -1,6 +1,10 @@
-let { is } = require('../../../lib')
 let { aliases, runtimeList } = require('lambda-runtimes')
+let { deepFrozenCopy } = require('@architect/utils')
+let { is, validationPatterns } = require('../../../lib')
+let { looserName } = validationPatterns
 let allRuntimes = runtimeList.concat([ 'deno', ...Object.keys(aliases) ])
+let validTypes = [ 'transpiled', 'compiled', 'interpreted' ]
+let builtTypes = validTypes.filter(t => t !== 'interpreted')
 
 module.exports = function setRuntimePlugins (params, project) {
   let { errors, inventory } = params
@@ -10,45 +14,52 @@ module.exports = function setRuntimePlugins (params, project) {
       runtimes: [],
     }
     // inventory._project is not yet built, so provide as much as we can to plugins for now
-    let inv = { ...inventory, _project: project }
+    let inv = deepFrozenCopy({ ...inventory, _project: project })
     let build
     runtimePlugins.forEach(fn => {
       let errType = `plugin: ${fn.plugin}, method: set.runtimes`
       try {
-        let result = fn({ inventory: inv })
-        result = is.array(result) ? result : [ result ]
-        result.forEach(runtime => {
-          // TODO add more validation
-          let { name } = runtime
-          if (!name) {
-            let msg = `Runtime plugin must provide a name: ${errType}`
-            return errors.push(msg)
-          }
-          if (allRuntimes.includes(name)) {
-            let msg = `Runtime name '${name}' is reserved: ${errType}`
-            return errors.push(msg)
-          }
-          if (runtimes[name]) {
-            let msg = `Runtime '${name}' already registered: ${errType}`
-            return errors.push(msg)
-          }
-          if (runtime.build) {
-            if (build && build !== runtime.build) {
-              errors.push(`Runtime '${name}' cannot set a build directory, as it is already configured to: ${build}`)
-            }
-            else if (is.bool(runtime.build) ||
-                !is.string(runtime.build)) {
-              build = 'build'
-            }
-            else build = runtime.build
-          }
-          runtimes.runtimes.push(name)
-          runtimes[name] = runtime
-        })
+        var result = fn({ arc: inv._project.arc, inventory: { inv } })
       }
       catch (err) {
-        errors.push(`Runtime plugin '${fn.plugin}' failed: ${err.message}`)
+        err.message = `Runtime plugin exception: ${errType}`
+                      + `\n` + err.message
+        throw err
       }
+      // Accept one or more results, then loop through them
+      result = is.array(result) ? result : [ result ]
+      result.forEach(runtime => {
+        let { name, type, baseRuntime } = runtime
+        if (!name || !looserName.test(name)) {
+          let msg = `Runtime plugin must provide a valid name: ${errType}`
+          return errors.push(msg)
+        }
+        if (!type || !validTypes.includes(type)) {
+          let msg = `Runtime plugin must provide a valid type: ${errType}`
+          return errors.push(msg)
+        }
+        if (allRuntimes.includes(name)) {
+          let msg = `Runtime name '${name}' is reserved: ${errType}`
+          return errors.push(msg)
+        }
+        if (runtimes[name]) {
+          let msg = `Runtime name '${name}' already registered: ${errType}`
+          return errors.push(msg)
+        }
+        if (builtTypes.includes(type)) {
+          if (build && runtime.build && build !== runtime.build) {
+            return errors.push(`Runtime '${name}' cannot set a build directory, as it is already configured to: ${build}`)
+          }
+          // Adhere to Postel's Law
+          build = 'build'
+          if (is.string(runtime.build)) build = runtime.build
+        }
+        if (type === 'transpiled' && !allRuntimes.includes(baseRuntime)) {
+          return errors.push(`Runtime '${name}' must include a valid baseRuntime property corresponding to a valid Lambda runtime (e.g. 'nodejs14.x')`)
+        }
+        runtimes.runtimes.push(name)
+        runtimes[name] = runtime
+      })
     })
     return { build, runtimes }
   }

package/src/config/project/prefs/dotenv.js

@@ -0,0 +1,81 @@
+// Copyright (c) 2015, Scott Motte
+// All rights reserved.
+
+/* istanbul ignore file */
+/* eslint-disable */
+// node_modules/dotenv/lib/main.js
+var fs = require("fs");
+var path = require("path");
+var os = require("os");
+var LINE = /(?:^|^)\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/mg;
+function parse(src) {
+  const obj = {};
+  let lines = src.toString();
+  lines = lines.replace(/\r\n?/mg, "\n");
+  let match;
+  while ((match = LINE.exec(lines)) != null) {
+    const key = match[1];
+    let value = match[2] || "";
+    value = value.trim();
+    const maybeQuote = value[0];
+    value = value.replace(/^(['"])([\s\S]+)\1$/mg, "$2");
+    if (maybeQuote === '"') {
+      value = value.replace(/\\n/g, "\n");
+      value = value.replace(/\\r/g, "\r");
+    }
+    obj[key] = value;
+  }
+  return obj;
+}
+function _log(message) {
+  console.log(`[dotenv][DEBUG] ${message}`);
+}
+function _resolveHome(envPath) {
+  return envPath[0] === "~" ? path.join(os.homedir(), envPath.slice(1)) : envPath;
+}
+function config(options) {
+  let dotenvPath = path.resolve(process.cwd(), ".env");
+  let encoding = "utf8";
+  const debug = Boolean(options && options.debug);
+  const override = Boolean(options && options.override);
+  if (options) {
+    if (options.path != null) {
+      dotenvPath = _resolveHome(options.path);
+    }
+    if (options.encoding != null) {
+      encoding = options.encoding;
+    }
+  }
+  try {
+    const parsed = DotenvModule.parse(fs.readFileSync(dotenvPath, { encoding }));
+    Object.keys(parsed).forEach(function(key) {
+      if (!Object.prototype.hasOwnProperty.call(process.env, key)) {
+        process.env[key] = parsed[key];
+      } else {
+        if (override === true) {
+          process.env[key] = parsed[key];
+        }
+        if (debug) {
+          if (override === true) {
+            _log(`"${key}" is already defined in \`process.env\` and WAS overwritten`);
+          } else {
+            _log(`"${key}" is already defined in \`process.env\` and was NOT overwritten`);
+          }
+        }
+      }
+    });
+    return { parsed };
+  } catch (e) {
+    if (debug) {
+      _log(`Failed to load ${dotenvPath} ${e.message}`);
+    }
+    return { error: e };
+  }
+}
+var DotenvModule = {
+  config,
+  parse
+};
+module.exports.config = DotenvModule.config;
+module.exports.parse = DotenvModule.parse;
+module.exports = DotenvModule;

package/src/config/project/{prefs.js → prefs/index.js}

@@ -1,6 +1,9 @@
-let read = require('../../read')
-let validate = require('./validate')
-let { is } = require('../../lib')
+let { join } = require('path')
+let { existsSync, readFileSync } = require('fs')
+let read = require('../../../read')
+let validate = require('../validate')
+let { is, validationPatterns: valid } = require('../../../lib')
+let { parse } = require('./dotenv')
 let { homedir } = require('os')
 
 module.exports = function getPrefs ({ scope, inventory, errors }) {
@@ -8,16 +11,20 @@ module.exports = function getPrefs ({ scope, inventory, errors }) {
     ? homedir()
     : inventory._project.cwd
 
-  // Populate preferences
+  let envFilepath = join(cwd, '.env')
+  let hasEnvFile = scope === 'local' && existsSync(envFilepath)
   let prefs = read({ type: 'preferences', cwd, errors })
-  if (prefs.filepath) {
-    let preferences = {}
+
+  if (!prefs.filepath && !hasEnvFile) return null
+
+  let preferences = {}
+
+  // Populate Architect preferences
+  if (prefs.filepath && prefs.arc) {
     // Ok, this gets a bit hairy
     // Arc outputs an object of nested arrays
     // Basically, construct a pared-down intermediate prefs obj for consumers
     Object.entries(prefs.arc).forEach(([ key, val ]) => {
-      // TODO add additional preferences checks and normalization
-
       /* istanbul ignore else: Parser should get this, but jic */
       if (!preferences[key]) preferences[key] = {}
       /* istanbul ignore else: Parser should only produce arrays, but jic */
@@ -42,9 +49,13 @@ module.exports = function getPrefs ({ scope, inventory, errors }) {
           /* istanbul ignore else: Yet another jic */
           if (preferences.env[e]) {
             Object.entries(preferences.env[e]).forEach(([ key, val ]) => {
+              if (!valid.envVar.test(key)) {
+                errors.push(`Env var '${key}' is invalid, must be [a-zA-Z0-9_]`)
+              }
               if (is.array(val)) preferences.env[e][key] = val.join(' ')
             })
           }
+          else preferences.env[e] = null
         })
       }
       // Turn Sandbox scripts into commands
@@ -58,16 +69,24 @@ module.exports = function getPrefs ({ scope, inventory, errors }) {
     })
 
     validate(preferences, errors)
+  }
 
-    return {
-      preferences: {
-        ...preferences,
-        _arc: prefs.arc,
-        _raw: prefs.raw,
-      },
-      preferencesFile: prefs.filepath
+  // Populate .env (testing environment only, disables other env vars)
+  if (hasEnvFile) {
+    let dotenv = parse(readFileSync(envFilepath))
+    preferences.env = {
+      testing: Object.keys(dotenv).length ? dotenv : null,
+      staging: null,
+      production: null,
     }
   }
 
-  return null
+  return {
+    preferences: {
+      ...preferences,
+      _arc: prefs.arc,
+      _raw: prefs.raw,
+    },
+    preferencesFile: prefs.filepath
+  }
 }

package/src/config/project/validate/index.js

@@ -2,12 +2,19 @@ let { is } = require('./../../../lib')
 
 module.exports = function validatePreferences (preferences, errors) {
   // Env checks
-  let { env } = preferences
-  if (!env) return
-  if (env && !is.object(env)) errors.push(`Invalid preferences setting: @env ${env}`)
-
+  let { env, sandbox } = preferences
   let envs = [ 'testing', 'staging', 'production' ]
-  envs.forEach(e => {
-    if (env[e] && !is.object(env[e])) errors.push(`Invalid preferences setting: @env ${e}`)
-  })
+
+  if (env && !is.object(env)) {
+    errors.push(`Invalid preferences setting: @env ${env}`)
+  }
+  else if (env) {
+    envs.forEach(e => {
+      if (env[e] && !is.object(env[e])) errors.push(`Invalid preferences setting: @env ${e}`)
+    })
+  }
+
+  if (sandbox?.env && !envs.includes(sandbox.env)) {
+    errors.push(`Invalid preferences setting: @sandbox env ${sandbox.env}`)
+  }
 }

package/src/defaults/index.js

@@ -7,7 +7,7 @@ let { pragmas } = require('../lib')
  * - Every possible officially supported value should be present
  */
 module.exports = function inventoryDefaults (params = {}) {
-  let { cwd = process.cwd(), region } = params
+  let { cwd = process.cwd(), deployStage = null, region } = params
   // Allow region env var override
   region = region || process.env.AWS_REGION || 'us-west-2'
   let defaultFunctionConfig = fnConfig()
@@ -16,7 +16,8 @@ module.exports = function inventoryDefaults (params = {}) {
     _arc: {
       version: 'Unknown',     // @architect/architect semver (if installed)
       defaultFunctionConfig,  // Architect's default function config
-      pragmas,                // Registry of all + Lambda pragmas
+      pragmas,                // Registry of all, Lambda, reserved, retired pragmas
+      deployStage,            // Deploy stage of the running project (if specified)
     },
     _project: {
       type: 'aws',
@@ -24,14 +25,18 @@ module.exports = function inventoryDefaults (params = {}) {
       src: join(cwd, 'src'),        // Default source tree dir
       build: null,                  // Optional build artifact dir
       manifest: null,               // Root project manifest filename
-      preferences: null,            // Realized preferences obj, resolved from global > local
+      preferences: null,            // Realized preferences obj, resolved from local > global
       localPreferences: null,       // Local preferences obj
       localPreferencesFile: null,   // Local preferences file path
       globalPreferences: null,      // Global preferences obj
       globalPreferencesFile: null,  // Global preferences file path
       defaultFunctionConfig,        // Project-level function config
       rootHandler: null,            // null | configured | arcStaticAssetProxy | proxy
-      env: null,                    // Env vars pulled from SSM + plugins
+      env: {                        // Env vars pulled from:
+        local: null,                // - Local/global prefs or .env
+        plugins: null,              // - Plugins
+        aws: null,                  // - SSM
+      },
       customRuntimes: null,         // Runtime plugins
       arc: [],                      // Raw arc obj
       raw: '',                      // Raw arc string

package/src/env/index.js

@@ -1,76 +1,91 @@
+let { mergeEnvVars } = require('../lib')
+
 /**
  * Read env vars out of SSM
  */
 module.exports = function env (params, inventory, callback) {
-  if (params.env) {
-    /* istanbul ignore next */
-    try {
-      // eslint-disable-next-line
-      var aws = require('aws-sdk')
-    }
-    catch (err) {
-      let msg = `'aws-sdk' not found, please install locally or globally (see also readme#aws-sdk-caveat)`
-      return callback(Error(msg))
-    }
-    let name = inventory.app
-    let { region } = inventory.aws
-    let ssm = new aws.SSM({ region })
-    let result = []
-
-    function getSomeEnvVars (name, NextToken, callback) {
-      // Base query to ssm
-      let query = {
-        Path: `/${name}`,
-        Recursive: true,
-        MaxResults: 10,
-        WithDecryption: true
-      }
+  if (!params.env) {
+    return callback()
+  }
 
-      // Check if we're paginating
-      /* istanbul ignore if */
-      if (NextToken) query.NextToken = NextToken
+  /* istanbul ignore next */
+  try {
+    // eslint-disable-next-line
+      var aws = require('aws-sdk')
+  }
+  catch (err) {
+    let msg = `'aws-sdk' not found, please install locally or globally (see also readme#aws-sdk-caveat)`
+    return callback(Error(msg))
+  }
+  let name = inventory.app
+  let { region } = inventory.aws
+  let ssm = new aws.SSM({ region })
+  let result = []
 
-      // Perform the query
-      ssm.getParametersByPath(query, function _query (err, data) {
-        if (err) callback(err)
-        else {
-          // Tidy up the response
-          result = result.concat(data.Parameters.map(function (param) {
-            let bits = param.Name.split('/')
-            return {
-              app: name, // jic
-              env: bits[2],
-              name: bits[3],
-              value: param.Value,
-            }
-          }))
-          // Check for more data and, if so, recurse
-          /* istanbul ignore if: Sadly no way to easily mock this for testing */
-          if (data.NextToken) {
-            getSomeEnvVars(name, data.NextToken, callback)
-          }
-          else callback(null, result)
-        }
-      })
+  function getSomeEnvVars (name, NextToken, callback) {
+    // Base query to ssm
+    let query = {
+      Path: `/${name}`,
+      Recursive: true,
+      MaxResults: 10,
+      WithDecryption: true
     }
 
-    getSomeEnvVars(name, false, function done (err, result) {
+    // Check if we're paginating
+    /* istanbul ignore if */
+    if (NextToken) query.NextToken = NextToken
+
+    // Perform the query
+    ssm.getParametersByPath(query, function _query (err, data) {
       if (err) callback(err)
       else {
-        if (result.length) {
-          let testing = null
-          let staging = null
-          let production = null
-          result.forEach(r => {
-            if (r.env === 'testing') testing = Object.assign({}, testing, { [r.name]: r.value })
-            if (r.env === 'staging') staging = Object.assign({}, staging, { [r.name]: r.value })
-            if (r.env === 'production') production = Object.assign({}, production, { [r.name]: r.value })
-          })
-          inventory._project.env = { testing, staging, production }
+        // Tidy up the response
+        result = result.concat(data.Parameters.map(function (param) {
+          let bits = param.Name.split('/')
+          return {
+            app: name, // jic
+            env: bits[2],
+            name: bits[3],
+            value: param.Value,
+          }
+        }))
+        // Check for more data and, if so, recurse
+        /* istanbul ignore if: Sadly no way to easily mock this for testing */
+        if (data.NextToken) {
+          getSomeEnvVars(name, data.NextToken, callback)
         }
-        callback()
+        else callback(null, result)
       }
     })
   }
-  else callback()
+
+  getSomeEnvVars(name, false, function done (err, result) {
+    if (err) callback(err)
+    else {
+      let testing = null
+      let staging = null
+      let production = null
+      if (result.length) {
+        // TODO refactor into a reducer?
+        result.forEach(({ env, name: k, value: v }) => {
+          if (env === 'testing') testing = Object.assign({}, testing, { [k]: v })
+          if (env === 'staging') staging = Object.assign({}, staging, { [k]: v })
+          if (env === 'production') production = Object.assign({}, production, { [k]: v })
+        })
+      }
+
+      let errors = []
+      inventory._project.env.aws = mergeEnvVars({
+        env: 'Application',
+        source: inventory._project.env.plugins,
+        target: { testing, staging, production },
+        errors,
+      })
+      if (errors.length) {
+        callback(Error(errors[0]))
+        return
+      }
+      callback()
+    }
+  })
 }

package/src/lib/asap-src.js

@@ -17,7 +17,7 @@ module.exports = function asapSrc () {
   try {
     return require.resolve('@architect/asap')
   }
-  catch (err) { /* Swallow */ }
-
-  throw Error('Cannot find ASAP module!')
+  catch (err) {
+    throw Error('Cannot find ASAP module!')
+  }
 }

package/src/lib/get-lambda-dirs.js

@@ -1,4 +1,4 @@
-let { join } = require('path')
+let { isAbsolute, join } = require('path')
 
 /**
  * Get the src (and build) dirs for a Lambda
@@ -29,8 +29,8 @@ function getLambdaDirs (params, options) {
 }
 
 function normalizeSrc (cwd, dir) {
-  if (!dir.startsWith(cwd)) return join(cwd, dir)
-  return dir
+  if (isAbsolute(dir)) return dir
+  return join(cwd, dir)
 }
 
 getLambdaDirs.normalizeSrc = normalizeSrc

package/src/lib/index.js

@@ -2,6 +2,7 @@ let asapSrc = require('./asap-src')
 let errorFmt = require('./error-fmt')
 let getLambdaDirs = require('./get-lambda-dirs')
 let is = require('./is')
+let mergeEnvVars = require('./merge-env-vars')
 let pragmas = require('./pragmas')
 
 /**
@@ -14,13 +15,28 @@ let compiledRuntimes = [ 'compiled', 'transpiled' ]
 // `any` must come last for Sandbox route sorting purposes
 let httpMethods = [ 'get', 'post', 'put', 'patch', 'delete', 'options', 'head', 'any' ]
 
+let validationPatterns = {
+  strictName: /^[a-z][a-z0-9-]+$/,
+  looseName: /^[a-z][a-zA-Z0-9-_]+$/,
+  looserName: /^[a-z][a-zA-Z0-9-._]+$/,
+  veryLooseName: /^[a-zA-Z0-9/\-._]*$/,
+  // IEEE 1003.1-2001 does not allow lowercase, so consider this a compromise for the Windows folks in the house
+  envVar: /^[a-zA-Z0-9_]+$/,
+}
+
+// Error tidier: remove the error name, just print the message + stack data
+let tidyError = err => err.message + `\n` + err.stack.split('\n').slice(1).join('\n')
+
 module.exports = {
   asapSrc,
   compiledRuntimes,
   errorFmt,
   getLambdaDirs,
-  normalizeSrc: getLambdaDirs.normalizeSrc,
   httpMethods,
   is,
+  mergeEnvVars,
+  normalizeSrc: getLambdaDirs.normalizeSrc,
   pragmas,
+  tidyError,
+  validationPatterns,
 }

package/src/lib/merge-env-vars.js

@@ -0,0 +1,32 @@
+let envs = [ 'testing', 'staging', 'production' ]
+
+module.exports = function mergeEnvVars (params) {
+  let { name, source, target, errors } = params
+  if (source === null) return target
+  if (target === null) return source
+  let probs = []
+
+  // Deep copy to reset any potential refs
+  let merged = JSON.parse(JSON.stringify(target))
+  envs.forEach(env => {
+    if (!source[env]) return
+    Object.keys(source[env]).forEach(k => {
+      if (merged[env]?.[k]) {
+        probs.push(`'${env}' variable '${k}'`)
+      }
+      else {
+        if (!merged[env]) merged[env] = {}
+        merged[env][k] = source[env][k]
+      }
+    })
+  })
+
+  if (probs.length) {
+    let s = probs.length > 1 ? 's' : ''
+    let msg = `${name} env var${s} conflicts with plugin:\n` +
+              `- ${probs.join('\n- ')}`
+    errors.push(msg)
+  }
+
+  return merged
+}

package/src/lib/pragmas.js

@@ -36,6 +36,7 @@ module.exports = {
     'indexes',
     'macros',
     'slack',
+    'streams', // Never fully launched
     // Static types
     'css',
     'html',

package/src/read/index.js

@@ -5,7 +5,6 @@ let projectManifest = {
   arc: [ 'app.arc', '.arc' ],
   json: [ 'arc.json' ],
   yaml: [ 'arc.yaml', 'arc.yml' ],
-  toml: [ 'arc.toml' ],
   manifest: [ 'package.json' ],
   _default: `@app\napp-default\n@http\n@static`,
 }
@@ -15,13 +14,12 @@ let functionConfig = {
   arc: [ 'config.arc', '.arc-config' ],
   json: [ 'arc.json', 'arc-config.json' ],
   yaml: [ 'config.yaml', 'config.yml', 'arc-config.yaml', 'arc-config.yml' ],
-  toml: [ 'config.toml', 'arc-config.toml' ],
 }
 
 // Local preferences
 let preferences = {
   arc: [ 'preferences.arc', 'prefs.arc', '.preferences.arc', '.prefs.arc', ],
-  // TODO add json, yaml, toml later if folks want it?
+  // TODO add json, yaml later if folks want it?
 }
 
 let reads = { projectManifest, functionConfig, preferences }

package/src/read/reader.js

@@ -37,7 +37,7 @@ module.exports = function reader (reads, cwd, errors) {
             if (raw.trim() === '') return errors.push(`Empty file: ${f}`)
             arc = type === 'arc'
               ? parse(raw)
-              : parse[type](raw) // Parser has convenient json, yaml, toml methods!
+              : parse[type](raw) // Parser has convenient json + yaml methods!
           }
           else {
             let pkg = JSON.parse(read(file))

package/src/validate/index.js

@@ -1,7 +1,8 @@
 let config = require('./config')
 let layers = require('./layers')
 let tablesChildren = require('./tables-children')
-let errorFmt = require('../lib/error-fmt')
+let paths = require('./paths')
+let { errorFmt } = require('../lib')
 
 /**
  * Final inventory validation
@@ -11,26 +12,32 @@ module.exports = function finalValidation (params, inventory) {
 
   /**
    * Deal with vendor configuration errors
+   * - Analyze function configuration
+   * - Ensure layer configuration will work, AWS blows up with awful errors on this
+   * - TODO add deeper policy validation
    */
-  // Analyze function configuration
   config(params, inventory, errors)
-
-  // Ensure layer configuration will work, AWS blows up with awful errors on this
   layers(params, inventory, errors)
-
-  // TODO add deeper policy validation here
-
   if (errors.length) {
     return errorFmt({ type: 'configuration', errors })
   }
 
   /**
    * Deal with project validation errors
+   * - Ensure @tables children (@tables-streams, @tables-indexes) have parent tables present
    */
-  // Ensure @tables children (@tables-streams, @tables-indexes) have parent tables present
   tablesChildren(inventory, errors)
-
   if (errors.length) {
     return errorFmt({ type: 'validation', errors })
   }
+
+  /**
+   * File path validation
+   * - Ensure all file paths are ascii
+   */
+  paths(inventory, errors)
+  if (errors.length) {
+    return errorFmt({ type: 'file path', errors })
+  }
+
 }

package/src/validate/paths.js

@@ -0,0 +1,17 @@
+module.exports = function checkFilePaths (inventory, errors) {
+  let ascii = /^[ -~]+$/
+  let err = str => errors.push(`${str} path must contain only ascii characters`)
+
+  let { _project: proj } = inventory
+  if (!ascii.test(proj.cwd)) return err('Project file')
+  if (!ascii.test(proj.src)) return err('Project source')
+  if (proj.build && !ascii.test(proj.build)) return err('Build')
+
+  let lambdas = inventory.lambdasBySrcDir
+  if (lambdas){
+    Object.values(lambdas).forEach(lambda => {
+      let { name, pragma, src } = lambda
+      if (!ascii.test(src)) err(`@${pragma} ${name} source`)
+    })
+  }
+}