@architect/inventory 3.0.0-RC.1 → 3.0.0-RC.5

package/changelog.md

@@ -6,14 +6,23 @@
 
 ### Added
 
+- Architect 10 plugin API support! Specifically:
+  - `plugins.set.runtimes` - custom runtime support (still in beta)
+  - `plugins.set.env` - add environment variables to all Lambdas
+  - `plugins.set.events|http|scheduled|tables-streams|ws` - generate or drop in Lambdas in Architect pragmas
+  - `plugins.set.customLambdas` - generate or drop in unique Lambdas with custom event sources
+  - More below...
 - Added `inv|get.plugins` tree + methods
   - What used to be `plugins` in the plugins beta is now `customLambdas` (see next item)
 - Added `inv|get.customLambdas`
   - Formerly `inv|get.plugins`
 - Added `inv._project.customRuntimes`
-- Added the concept of a `build` destination (currently only available via plugin)
+- Added low-level support for `build` destinations to runtime plugins that register type `transpiled` or `compiled`
 - Added `handlerModuleSystem` property for `nodejs14.x` Lambdas, with a value of `cjs` or `esm` based on Lambda + Node.js conventions
 - Added `handlerFile` detection for `nodejs14.x` + `deno` Lambdas
+  - This will detect the correct handler file on the filesystem, and fall back to a default handler file if none are found (e.g. `index.js` in `nodejs14.x`)
+- Added `inv._arc.deployStage` property, enabling Inventory to be aware of an intended deploy stage; (this property may change, consider it in beta!)
+- Added built-in support for reading `.env` files when enumerating local env var preferences
 
 
 ### Changed
@@ -21,15 +30,19 @@
 - Breaking change: changed `_project.src`, added `_project.cwd`, making both the pair significantly more literal and descriptive
   - `_project.src` is now the default source tree folder (eg `$cwd/src`)
   - `_project.cwd` refers to the current working directory of the project
+- Breaking change: `_project.env` is now by default an object populated by three properties: `local`, `plugins`, and `aws`, reflecting the env vars found for each environment
 - Breaking change: AWS region prioritizes a region passed via param over `AWS_REGION` env var; this should realistically have little or no effect in practice
 - Breaking change: legacy `@tables-streams` folders (`src/tables/...` and `src/streams/...`) are now deprecated
   - Existing functions can be simply moved to `src/tables-streams/{name}` (or use a custom `src` property)
 - Breaking change: renamed `lambda.handlerFunction` to `lambda.handlerMethod`
-- Breaking change: prioritize mod.ts|js in Deno
-- Internal change: performance improvements to building `inv.shared` + `inv.views`
+- Breaking change: prioritize `mod.ts|js` handlers in Deno Lambdas
+- Breaking change: removed `toml` support
+- Performance improvements to building `inv.shared` + `inv.views`
 - Improved memory footprint of Inventory object by preserving references in `lambdaSrcDirs`, `lambdasBySrcDir`
-  - Added `pragma` property to all Lambdas to preserve references
+  - Added `pragma` property to all Lambdas to aid in reference preservation
 - Tidy up order of enumerated properties in each Lambda
+Update CI
+- Stop publishing to the GitHub Package registry
 
 ---
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@architect/inventory",
-  "version": "3.0.0-RC.1",
+  "version": "3.0.0-RC.5",
   "description": "Architect project resource enumeration utility",
   "main": "src/index.js",
   "scripts": {
@@ -9,7 +9,8 @@
     "test:integration": "cross-env tape 'test/integration/**/*-test.js' | tap-spec",
     "coverage": "nyc --reporter=lcov --reporter=text npm run test:unit",
     "lint": "eslint . --fix",
-    "rc": "npm version prerelease --preid RC"
+    "rc": "npm version prerelease --preid RC",
+    "vendor": "scripts/vendor"
   },
   "engines": {
     "node": ">=14"
@@ -20,22 +21,23 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
-    "@architect/asap": "~4.1.0",
-    "@architect/parser": "~5.0.2",
+    "@architect/asap": "~5.0.0-RC.0",
+    "@architect/parser": "~6.0.0-RC.0",
     "@architect/utils": "~3.0.4",
     "lambda-runtimes": "~1.1.1"
   },
   "devDependencies": {
     "@architect/eslint-config": "~2.0.1",
-    "aws-sdk": "2.880.0",
-    "aws-sdk-mock": "~5.5.0",
+    "aws-sdk": "2.1001.0",
+    "aws-sdk-mock": "~5.6.0",
     "cross-env": "~7.0.3",
-    "eslint": "~8.5.0",
+    "dotenv": "~14.3.2",
+    "eslint": "~8.7.0",
     "mock-fs": "~5.1.2",
     "mock-require": "~3.0.3",
     "nyc": "~15.1.0",
     "tap-spec": "^5.0.0",
-    "tape": "^5.3.2"
+    "tape": "^5.4.1"
   },
   "eslintConfig": {
     "extends": "@architect/eslint-config"

package/readme.md

@@ -43,7 +43,7 @@ The inventory object contains the entirety of a project's data, including Archit
 
 Top-level inventory parameters that start with an underscore (e.g. `_arc`, `_project`) denote project metadata or internal diagnostic data; all other parameters represent userland project resources.
 
-In a project inventory, `null` values are used as placeholders for known values or options that were not user-defined. The existence of a non-`null` value can be inferred as a user having specifically defined a setting. For example: `arc.http === null` can be construed as the user having **not** defined an `@http` pragma. This rule has some exceptions:
+In a project inventory, `null` values are used as placeholders for known values or options that were not user-defined. The existence of a non-`null` value can be inferred as a user having specifically defined a setting. For example: `arc.http: null` can be construed as the user having **not** defined an `@http` pragma. This rule has some exceptions:
 
 - A handful of settings that must be backfilled if not supplied
   - Example: `inv.aws.region`, which is required by the `aws-sdk` to function, and will be backfilled if not defined
@@ -51,9 +51,10 @@ In a project inventory, `null` values are used as placeholders for known values
   - Example: while `@static` can be defined on its own without any other pragmas, the existence of `@http` infers `@static`
   - Thus, the act of adding `@http` will necessarily make `inv.static` non-`null`
 - Settings that generate related resources
-  - Example: DynamoDB streams can be defined in `@tables` with `stream true`; Inventory would interpret a table with `stream true` as a new `inv.streams` resource and thus make `inv.streams` non-`null`
-
-> Note: The `inv` format is primarily designed and intended for internal use within Architect libraries; as such, Inventory may theoretically introduce breaking changes in its behavior, shape, or naming conventions.
+  - Example: DynamoDB streams can be defined in `@tables` with `stream true`; Inventory would interpret a table with `stream true` as a new `inv['tables-streams']` resource and thus make `inv['tables-streams']` non-`null`
+- Lambda `handlerFile` file path property is present even if the file is not
+  - This differs from Lambda `configFile` file path properties, which will be `null` if no file is present
+  - This exception is namely because some workflows may need the computed default handler path (example: when running `arc create`)
 
 
 #### `get`

package/src/config/arc.js

@@ -6,14 +6,14 @@ let { join } = require('path')
  */
 module.exports = function getArcConfig (params) {
   let { cwd, inventory } = params
-  let arc = { ...inventory._arc }
+  let _arc = { ...inventory._arc }
 
   // Version
   let installed = join(cwd, 'node_modules', '@architect', 'architect', 'package.json')
   if (existsSync(installed)) {
     let { version } = JSON.parse(readFileSync(installed))
-    arc.version = version
+    _arc.version = version
   }
 
-  return arc
+  return _arc
 }

package/src/config/project/index.js

@@ -2,25 +2,25 @@ let { join } = require('path')
 let upsert = require('../_upsert')
 let prefs = require('./prefs')
 let plugins = require('./plugins')
-let { is } = require('../../lib')
+let { is, mergeEnvVars } = require('../../lib')
 
 /**
  * Get the project-level configuration, overlaying arc.aws settings (if present)
  */
 module.exports = function getProjectConfig (params) {
   let { arc, errors, raw, filepath, inventory } = params
-  let project = {
+  let _project = {
     ...inventory._project,
     arc,
     raw,
   }
 
   if (arc.aws) {
-    project.defaultFunctionConfig = upsert(project.defaultFunctionConfig, arc.aws)
+    _project.defaultFunctionConfig = upsert(_project.defaultFunctionConfig, arc.aws)
   }
 
   if (filepath) {
-    project.manifest = filepath
+    _project.manifest = filepath
   }
 
   // Parse local and global project preferences
@@ -29,37 +29,48 @@ module.exports = function getProjectConfig (params) {
     let p = prefs({ scope, inventory, errors })
     if (p) {
       // Set up the scoped metadata
-      project[`${scope}Preferences`] = p.preferences
-      project[`${scope}PreferencesFile`] = p.preferencesFile
+      _project[`${scope}Preferences`] = p.preferences
+      _project[`${scope}PreferencesFile`] = p.preferencesFile
 
       // Build out the final preferences
       /* istanbul ignore else: jic */
-      if (!project.preferences) project.preferences = {}
+      if (!_project.preferences) _project.preferences = {}
       Object.keys(p.preferences).forEach(pragma => {
         // Ignore the raw data
         if (pragma === '_arc' || pragma === '_raw') return
         // Allow booleans, etc.
         if (!is.object(p.preferences[pragma])) {
-          project.preferences[pragma] = p.preferences[pragma]
+          _project.preferences[pragma] = p.preferences[pragma]
           return
         }
         // Traverse and merge individual settings
         /* istanbul ignore else: jic */
-        if (!project.preferences[pragma]) project.preferences[pragma] = {}
+        if (!_project.preferences[pragma]) _project.preferences[pragma] = {}
         Object.entries(p.preferences[pragma]).forEach(([ setting, value ]) => {
-          project.preferences[pragma][setting] = value
+          _project.preferences[pragma][setting] = value
         })
       })
     }
   }
 
+  // Populate local env from preferences
+  if (_project.preferences?.env) {
+    _project.env.local = _project.preferences.env
+  }
+
   if (inventory.plugins?._methods) {
-    // TODO: project.env = plugins.env(params, project)
+    _project.env.plugins = plugins.env(params, _project)
+    _project.env.local = mergeEnvVars({
+      name: 'Local',
+      source: _project.env.plugins,
+      target: _project.env.local,
+      errors,
+    })
 
-    let { build, runtimes } = plugins.runtimes(params, project)
-    if (build)    project.build = join(project.cwd, build)
-    if (runtimes) project.customRuntimes = runtimes
+    let { build, runtimes } = plugins.runtimes(params, _project)
+    if (build)    _project.build = join(_project.cwd, build)
+    if (runtimes) _project.customRuntimes = runtimes
   }
 
-  return project
+  return _project
 }

package/src/config/project/plugins/env.js

@@ -0,0 +1,33 @@
+let { is } = require('../../../lib')
+
+module.exports = function setEnvPlugins (params, project) {
+  let { errors, inventory } = params
+  let envPlugins = inventory.plugins?._methods?.set?.env
+  if (envPlugins?.length) {
+    let env = {}
+
+    // inventory._project is not yet built, so provide as much as we can to plugins for now
+    let inv = { ...inventory, _project: project }
+    envPlugins.forEach(fn => {
+      let errType = `plugin: ${fn.plugin}, method: set.env`
+      try {
+        let result = fn({ inventory: { inv } })
+        if (!is.object(result) || !Object.keys(result).length) {
+          return errors.push(`Env plugin returned invalid data, must return an Object with one or more keys + values: ${errType}`)
+        }
+        Object.entries(result).forEach(([ k, v ]) => {
+          if (env[k]) {
+            return errors.push(`Env var '${k}' already registered: ${errType}`)
+          }
+          if (is.object(v) || is.array(v)) env[k] = JSON.stringify(v)
+          else env[k] = String(v)
+        })
+      }
+      catch (err) {
+        errors.push(`Runtime plugin '${fn.plugin}' failed: ${err.message}`)
+      }
+    })
+    return { testing: env, staging: env, production: env }
+  }
+  return inventory._project.env.plugins
+}

package/src/config/project/plugins/index.js

@@ -1,5 +1,7 @@
+let env = require('./env')
 let runtimes = require('./runtimes')
 
 module.exports = {
+  env,
   runtimes,
 }

package/src/config/project/plugins/runtimes.js

@@ -15,7 +15,7 @@ module.exports = function setRuntimePlugins (params, project) {
     runtimePlugins.forEach(fn => {
       let errType = `plugin: ${fn.plugin}, method: set.runtimes`
       try {
-        let result = fn({ inventory: inv })
+        let result = fn({ inventory: { inv } })
         result = is.array(result) ? result : [ result ]
         result.forEach(runtime => {
           // TODO add more validation

package/src/config/project/prefs/dotenv.js

@@ -0,0 +1,112 @@
+// Copyright (c) 2015, Scott Motte
+// All rights reserved.
+
+/* istanbul ignore file */
+/* eslint-disable */
+// node_modules/dotenv/lib/main.js
+var fs = require("fs");
+var path = require("path");
+var os = require("os");
+function log(message) {
+  console.log(`[dotenv][DEBUG] ${message}`);
+}
+var NEWLINE = "\n";
+var RE_INI_KEY_VAL = /^\s*([\w.-]+)\s*=\s*("[^"]*"|'[^']*'|.*?)(\s+#.*)?$/;
+var RE_NEWLINES = /\\n/g;
+var NEWLINES_MATCH = /\r\n|\n|\r/;
+function parse(src, options) {
+  const debug = Boolean(options && options.debug);
+  const multiline = Boolean(options && options.multiline);
+  const obj = {};
+  const lines = src.toString().split(NEWLINES_MATCH);
+  for (let idx = 0; idx < lines.length; idx++) {
+    let line = lines[idx];
+    const keyValueArr = line.match(RE_INI_KEY_VAL);
+    if (keyValueArr != null) {
+      const key = keyValueArr[1];
+      let val = keyValueArr[2] || "";
+      let end = val.length - 1;
+      const isDoubleQuoted = val[0] === '"' && val[end] === '"';
+      const isSingleQuoted = val[0] === "'" && val[end] === "'";
+      const isMultilineDoubleQuoted = val[0] === '"' && val[end] !== '"';
+      const isMultilineSingleQuoted = val[0] === "'" && val[end] !== "'";
+      if (multiline && (isMultilineDoubleQuoted || isMultilineSingleQuoted)) {
+        const quoteChar = isMultilineDoubleQuoted ? '"' : "'";
+        val = val.substring(1);
+        while (idx++ < lines.length - 1) {
+          line = lines[idx];
+          end = line.length - 1;
+          if (line[end] === quoteChar) {
+            val += NEWLINE + line.substring(0, end);
+            break;
+          }
+          val += NEWLINE + line;
+        }
+      } else if (isSingleQuoted || isDoubleQuoted) {
+        val = val.substring(1, end);
+        if (isDoubleQuoted) {
+          val = val.replace(RE_NEWLINES, NEWLINE);
+        }
+      } else {
+        val = val.trim();
+      }
+      obj[key] = val;
+    } else if (debug) {
+      const trimmedLine = line.trim();
+      if (trimmedLine.length && trimmedLine[0] !== "#") {
+        log(`Failed to match key and value when parsing line ${idx + 1}: ${line}`);
+      }
+    }
+  }
+  return obj;
+}
+function resolveHome(envPath) {
+  return envPath[0] === "~" ? path.join(os.homedir(), envPath.slice(1)) : envPath;
+}
+function config(options) {
+  let dotenvPath = path.resolve(process.cwd(), ".env");
+  let encoding = "utf8";
+  const debug = Boolean(options && options.debug);
+  const override = Boolean(options && options.override);
+  const multiline = Boolean(options && options.multiline);
+  if (options) {
+    if (options.path != null) {
+      dotenvPath = resolveHome(options.path);
+    }
+    if (options.encoding != null) {
+      encoding = options.encoding;
+    }
+  }
+  try {
+    const parsed = DotenvModule.parse(fs.readFileSync(dotenvPath, { encoding }), { debug, multiline });
+    Object.keys(parsed).forEach(function(key) {
+      if (!Object.prototype.hasOwnProperty.call(process.env, key)) {
+        process.env[key] = parsed[key];
+      } else {
+        if (override === true) {
+          process.env[key] = parsed[key];
+        }
+        if (debug) {
+          if (override === true) {
+            log(`"${key}" is already defined in \`process.env\` and WAS overwritten`);
+          } else {
+            log(`"${key}" is already defined in \`process.env\` and was NOT overwritten`);
+          }
+        }
+      }
+    });
+    return { parsed };
+  } catch (e) {
+    if (debug) {
+      log(`Failed to load ${dotenvPath} ${e.message}`);
+    }
+    return { error: e };
+  }
+}
+var DotenvModule = {
+  config,
+  parse
+};
+module.exports.config = DotenvModule.config;
+module.exports.parse = DotenvModule.parse;
+module.exports = DotenvModule;

package/src/config/project/{prefs.js → prefs/index.js}

@@ -1,6 +1,9 @@
-let read = require('../../read')
-let validate = require('./validate')
-let { is } = require('../../lib')
+let { join } = require('path')
+let { existsSync, readFileSync } = require('fs')
+let read = require('../../../read')
+let validate = require('../validate')
+let { is } = require('../../../lib')
+let { parse } = require('./dotenv')
 let { homedir } = require('os')
 
 module.exports = function getPrefs ({ scope, inventory, errors }) {
@@ -8,10 +11,16 @@ module.exports = function getPrefs ({ scope, inventory, errors }) {
     ? homedir()
     : inventory._project.cwd
 
-  // Populate preferences
+  let envFilepath = join(cwd, '.env')
+  let hasEnvFile = scope === 'local' && existsSync(envFilepath)
   let prefs = read({ type: 'preferences', cwd, errors })
+
+  if (!prefs.filepath && !hasEnvFile) return null
+
+  let preferences = {}
+
+  // Populate Architect preferences
   if (prefs.filepath) {
-    let preferences = {}
     // Ok, this gets a bit hairy
     // Arc outputs an object of nested arrays
     // Basically, construct a pared-down intermediate prefs obj for consumers
@@ -45,6 +54,7 @@ module.exports = function getPrefs ({ scope, inventory, errors }) {
               if (is.array(val)) preferences.env[e][key] = val.join(' ')
             })
           }
+          else preferences.env[e] = null
         })
       }
       // Turn Sandbox scripts into commands
@@ -58,16 +68,24 @@ module.exports = function getPrefs ({ scope, inventory, errors }) {
     })
 
     validate(preferences, errors)
+  }
 
-    return {
-      preferences: {
-        ...preferences,
-        _arc: prefs.arc,
-        _raw: prefs.raw,
-      },
-      preferencesFile: prefs.filepath
+  // Populate .env (testing environment only, disables other env vars)
+  if (hasEnvFile) {
+    let dotenv = parse(readFileSync(envFilepath))
+    preferences.env = {
+      testing: Object.keys(dotenv).length ? dotenv : null,
+      staging: null,
+      production: null,
     }
   }
 
-  return null
+  return {
+    preferences: {
+      ...preferences,
+      _arc: prefs.arc,
+      _raw: prefs.raw,
+    },
+    preferencesFile: prefs.filepath
+  }
 }

package/src/config/project/validate/index.js

@@ -2,12 +2,19 @@ let { is } = require('./../../../lib')
 
 module.exports = function validatePreferences (preferences, errors) {
   // Env checks
-  let { env } = preferences
-  if (!env) return
-  if (env && !is.object(env)) errors.push(`Invalid preferences setting: @env ${env}`)
-
+  let { env, sandbox } = preferences
   let envs = [ 'testing', 'staging', 'production' ]
-  envs.forEach(e => {
-    if (env[e] && !is.object(env[e])) errors.push(`Invalid preferences setting: @env ${e}`)
-  })
+
+  if (env && !is.object(env)) {
+    errors.push(`Invalid preferences setting: @env ${env}`)
+  }
+  else if (env) {
+    envs.forEach(e => {
+      if (env[e] && !is.object(env[e])) errors.push(`Invalid preferences setting: @env ${e}`)
+    })
+  }
+
+  if (sandbox?.env && !envs.includes(sandbox.env)) {
+    errors.push(`Invalid preferences setting: @sandbox env ${sandbox.env}`)
+  }
 }

package/src/defaults/index.js

@@ -7,7 +7,7 @@ let { pragmas } = require('../lib')
  * - Every possible officially supported value should be present
  */
 module.exports = function inventoryDefaults (params = {}) {
-  let { cwd = process.cwd(), region } = params
+  let { cwd = process.cwd(), deployStage = null, region } = params
   // Allow region env var override
   region = region || process.env.AWS_REGION || 'us-west-2'
   let defaultFunctionConfig = fnConfig()
@@ -16,7 +16,8 @@ module.exports = function inventoryDefaults (params = {}) {
     _arc: {
       version: 'Unknown',     // @architect/architect semver (if installed)
       defaultFunctionConfig,  // Architect's default function config
-      pragmas,                // Registry of all + Lambda pragmas
+      pragmas,                // Registry of all, Lambda, reserved, retired pragmas
+      deployStage,            // Deploy stage of the running project (if specified)
     },
     _project: {
       type: 'aws',
@@ -24,14 +25,18 @@ module.exports = function inventoryDefaults (params = {}) {
       src: join(cwd, 'src'),        // Default source tree dir
       build: null,                  // Optional build artifact dir
       manifest: null,               // Root project manifest filename
-      preferences: null,            // Realized preferences obj, resolved from global > local
+      preferences: null,            // Realized preferences obj, resolved from local > global
       localPreferences: null,       // Local preferences obj
       localPreferencesFile: null,   // Local preferences file path
       globalPreferences: null,      // Global preferences obj
       globalPreferencesFile: null,  // Global preferences file path
       defaultFunctionConfig,        // Project-level function config
       rootHandler: null,            // null | configured | arcStaticAssetProxy | proxy
-      env: null,                    // Env vars pulled from SSM + plugins
+      env: {                        // Env vars pulled from:
+        local: null,                // Local/global prefs or .env
+        plugins: null,              // Plugins
+        aws: null,                  // SSM
+      },
       customRuntimes: null,         // Runtime plugins
       arc: [],                      // Raw arc obj
       raw: '',                      // Raw arc string

package/src/env/index.js

@@ -1,76 +1,91 @@
+let { mergeEnvVars } = require('../lib')
+
 /**
  * Read env vars out of SSM
  */
 module.exports = function env (params, inventory, callback) {
-  if (params.env) {
-    /* istanbul ignore next */
-    try {
-      // eslint-disable-next-line
-      var aws = require('aws-sdk')
-    }
-    catch (err) {
-      let msg = `'aws-sdk' not found, please install locally or globally (see also readme#aws-sdk-caveat)`
-      return callback(Error(msg))
-    }
-    let name = inventory.app
-    let { region } = inventory.aws
-    let ssm = new aws.SSM({ region })
-    let result = []
-
-    function getSomeEnvVars (name, NextToken, callback) {
-      // Base query to ssm
-      let query = {
-        Path: `/${name}`,
-        Recursive: true,
-        MaxResults: 10,
-        WithDecryption: true
-      }
+  if (!params.env) {
+    return callback()
+  }
 
-      // Check if we're paginating
-      /* istanbul ignore if */
-      if (NextToken) query.NextToken = NextToken
+  /* istanbul ignore next */
+  try {
+    // eslint-disable-next-line
+      var aws = require('aws-sdk')
+  }
+  catch (err) {
+    let msg = `'aws-sdk' not found, please install locally or globally (see also readme#aws-sdk-caveat)`
+    return callback(Error(msg))
+  }
+  let name = inventory.app
+  let { region } = inventory.aws
+  let ssm = new aws.SSM({ region })
+  let result = []
 
-      // Perform the query
-      ssm.getParametersByPath(query, function _query (err, data) {
-        if (err) callback(err)
-        else {
-          // Tidy up the response
-          result = result.concat(data.Parameters.map(function (param) {
-            let bits = param.Name.split('/')
-            return {
-              app: name, // jic
-              env: bits[2],
-              name: bits[3],
-              value: param.Value,
-            }
-          }))
-          // Check for more data and, if so, recurse
-          /* istanbul ignore if: Sadly no way to easily mock this for testing */
-          if (data.NextToken) {
-            getSomeEnvVars(name, data.NextToken, callback)
-          }
-          else callback(null, result)
-        }
-      })
+  function getSomeEnvVars (name, NextToken, callback) {
+    // Base query to ssm
+    let query = {
+      Path: `/${name}`,
+      Recursive: true,
+      MaxResults: 10,
+      WithDecryption: true
     }
 
-    getSomeEnvVars(name, false, function done (err, result) {
+    // Check if we're paginating
+    /* istanbul ignore if */
+    if (NextToken) query.NextToken = NextToken
+
+    // Perform the query
+    ssm.getParametersByPath(query, function _query (err, data) {
       if (err) callback(err)
       else {
-        if (result.length) {
-          let testing = null
-          let staging = null
-          let production = null
-          result.forEach(r => {
-            if (r.env === 'testing') testing = Object.assign({}, testing, { [r.name]: r.value })
-            if (r.env === 'staging') staging = Object.assign({}, staging, { [r.name]: r.value })
-            if (r.env === 'production') production = Object.assign({}, production, { [r.name]: r.value })
-          })
-          inventory._project.env = { testing, staging, production }
+        // Tidy up the response
+        result = result.concat(data.Parameters.map(function (param) {
+          let bits = param.Name.split('/')
+          return {
+            app: name, // jic
+            env: bits[2],
+            name: bits[3],
+            value: param.Value,
+          }
+        }))
+        // Check for more data and, if so, recurse
+        /* istanbul ignore if: Sadly no way to easily mock this for testing */
+        if (data.NextToken) {
+          getSomeEnvVars(name, data.NextToken, callback)
         }
-        callback()
+        else callback(null, result)
       }
     })
   }
-  else callback()
+
+  getSomeEnvVars(name, false, function done (err, result) {
+    if (err) callback(err)
+    else {
+      let testing = null
+      let staging = null
+      let production = null
+      if (result.length) {
+        // TODO refactor into a reducer?
+        result.forEach(({ env, name: k, value: v }) => {
+          if (env === 'testing') testing = Object.assign({}, testing, { [k]: v })
+          if (env === 'staging') staging = Object.assign({}, staging, { [k]: v })
+          if (env === 'production') production = Object.assign({}, production, { [k]: v })
+        })
+      }
+
+      let errors = []
+      inventory._project.env.aws = mergeEnvVars({
+        env: 'Application',
+        source: inventory._project.env.plugins,
+        target: { testing, staging, production },
+        errors,
+      })
+      if (errors.length) {
+        callback(Error(errors[0]))
+        return
+      }
+      callback()
+    }
+  })
 }

package/src/lib/asap-src.js

@@ -17,7 +17,7 @@ module.exports = function asapSrc () {
   try {
     return require.resolve('@architect/asap')
   }
-  catch (err) { /* Swallow */ }
-
-  throw Error('Cannot find ASAP module!')
+  catch (err) {
+    throw Error('Cannot find ASAP module!')
+  }
 }

package/src/lib/index.js

@@ -2,6 +2,7 @@ let asapSrc = require('./asap-src')
 let errorFmt = require('./error-fmt')
 let getLambdaDirs = require('./get-lambda-dirs')
 let is = require('./is')
+let mergeEnvVars = require('./merge-env-vars')
 let pragmas = require('./pragmas')
 
 /**
@@ -19,8 +20,9 @@ module.exports = {
   compiledRuntimes,
   errorFmt,
   getLambdaDirs,
-  normalizeSrc: getLambdaDirs.normalizeSrc,
   httpMethods,
   is,
+  mergeEnvVars,
+  normalizeSrc: getLambdaDirs.normalizeSrc,
   pragmas,
 }

package/src/lib/merge-env-vars.js

@@ -0,0 +1,32 @@
+let envs = [ 'testing', 'staging', 'production' ]
+
+module.exports = function mergeEnvVars (params) {
+  let { name, source, target, errors } = params
+  if (source === null) return target
+  if (target === null) return source
+  let probs = []
+
+  // Deep copy to reset any potential refs
+  let merged = JSON.parse(JSON.stringify(target))
+  envs.forEach(env => {
+    if (!source[env]) return
+    Object.keys(source[env]).forEach(k => {
+      if (merged[env]?.[k]) {
+        probs.push(`'${env}' variable '${k}'`)
+      }
+      else {
+        if (!merged[env]) merged[env] = {}
+        merged[env][k] = source[env][k]
+      }
+    })
+  })
+
+  if (probs.length) {
+    let s = probs.length > 1 ? 's' : ''
+    let msg = `${name} env var${s} conflicts with plugin:\n` +
+              `- ${probs.join('\n- ')}`
+    errors.push(msg)
+  }
+
+  return merged
+}

package/src/lib/pragmas.js

@@ -36,6 +36,7 @@ module.exports = {
     'indexes',
     'macros',
     'slack',
+    'streams', // Never fully launched
     // Static types
     'css',
     'html',

package/src/read/index.js

@@ -5,7 +5,6 @@ let projectManifest = {
   arc: [ 'app.arc', '.arc' ],
   json: [ 'arc.json' ],
   yaml: [ 'arc.yaml', 'arc.yml' ],
-  toml: [ 'arc.toml' ],
   manifest: [ 'package.json' ],
   _default: `@app\napp-default\n@http\n@static`,
 }
@@ -15,13 +14,12 @@ let functionConfig = {
   arc: [ 'config.arc', '.arc-config' ],
   json: [ 'arc.json', 'arc-config.json' ],
   yaml: [ 'config.yaml', 'config.yml', 'arc-config.yaml', 'arc-config.yml' ],
-  toml: [ 'config.toml', 'arc-config.toml' ],
 }
 
 // Local preferences
 let preferences = {
   arc: [ 'preferences.arc', 'prefs.arc', '.preferences.arc', '.prefs.arc', ],
-  // TODO add json, yaml, toml later if folks want it?
+  // TODO add json, yaml later if folks want it?
 }
 
 let reads = { projectManifest, functionConfig, preferences }

package/src/read/reader.js

@@ -37,7 +37,7 @@ module.exports = function reader (reads, cwd, errors) {
             if (raw.trim() === '') return errors.push(`Empty file: ${f}`)
             arc = type === 'arc'
               ? parse(raw)
-              : parse[type](raw) // Parser has convenient json, yaml, toml methods!
+              : parse[type](raw) // Parser has convenient json + yaml methods!
           }
           else {
             let pkg = JSON.parse(read(file))