@foxtware/mineral 0.1.4 → 0.1.6

package/.gcloudignore

@@ -7,11 +7,8 @@
 #   $ gcloud topic gcloudignore
 #
 .gcloudignore
-# If you would like to upload your .git directory, .gitignore file or files
-# from your .gitignore file, remove the corresponding line
-# below:
 .git
 .gitignore
 
 node_modules
-#!include:.gitignore
+.creds.yml

package/README.md

@@ -51,4 +51,4 @@ The [bedrock](https://github.com/GorgonFreeman/bedrock) middleware, refactored f
 - **Monorepo structure**
 Mineral gets pushed to from a larger repo that can also contain private functions. Mineral should be strictly useful stuff for the public, and can be used standalone, but needs to be instantiated for serving, setting stuff like which creds file to use. This allows it to be used as part of another repo in the same HTTP/curl way as by itself. Pass `--workspace` to locate `.creds.yml` and `--api_dirs` to serve additional function directories.
 
-For cloud deploy, workspaces use `.hosting.yml` and `npm run host` (same `--workspace` / `--api_dirs` flags as dev/serve). See `.hosting.yml.sample`.
+For cloud deploy, workspaces use `hosting/.hosting.yml` and `npm run host` (same `--workspace` / `--api_dirs` flags as dev/serve). See `hosting/.hosting.yml.sample`.

package/_build_scripts/createNewFunction.js

@@ -3,31 +3,51 @@ const path = require('path');
 const { spawn } = require('child_process');
 const { askQuestion, capitaliseString } = require('../api/utils');
 
-const apiDirectory = path.join(__dirname, '../api');
-const rootExampleJsPath = path.join(apiDirectory, '_example.js');
+const mineralRoot = path.join(__dirname, '..');
+const mineralApiDirectory = path.join(mineralRoot, 'api');
+const rootExampleJsPath = path.join(mineralApiDirectory, '_example.js');
 
 const excludedDirs = new Set([
   'node_modules',
 ]);
 
-const printHelp = () => {
+const normalizePath = (dirPath) => dirPath.replace(/\/$/, '');
+
+const isMineralCwd = () => normalizePath(process.cwd()) === normalizePath(mineralRoot);
+
+const getContext = () => {
+  const inMineral = isMineralCwd();
+
+  return {
+    inMineral,
+    apiDirectory: inMineral ? mineralApiDirectory : path.join(process.cwd(), 'api'),
+    mineralApiDirectory,
+  };
+};
+
+const printHelp = (context) => {
+  const templateHelp = context.inMineral
+    ? `example template to copy (default: _example.js in dir, else api/_example.js)
+                  examples: _example.js, getsingle, _example.getsingle.js`
+    : `mineral example template to copy (default: _example.js)
+                  examples: _example.js, shopify/_example.js, shopify/getsingle`;
+
   console.log(`
 Usage:
   npm run new
   npm run new -- --dir <apiSubdir> --name <name> [--template <exampleTemplate>] [--commit]
 
 Options:
-  --dir, -d       api subdirectory (e.g. peoplevox, shopify)
+  --dir, -d       api subdirectory (e.g. peoplevox, shopify). Omit for flat api/ dirs.
   --name, -n      function name suffix (e.g. orderGet → peoplevoxOrderGet)
-  --template, -t  example template to copy (default: _example.js in dir, else api/_example.js)
-                  examples: _example.js, getsingle, _example.getsingle.js
+  --template, -t  ${ templateHelp }
   --commit        auto-commit the stub after creation
   --help, -h      show this help
 
 Examples:
   npm run new -- --dir peoplevox --name orderGet
-  npm run new -- --dir peoplevox --name orderGet --template _example.js
   npm run new -- --dir shopify --name pageGet --template getsingle
+  npm run new -- --name geodeBye --template shopify/_example.js
 `);
 };
 
@@ -124,11 +144,51 @@ const findExampleFiles = async (dirPath) => {
   }
 };
 
+const collectMineralExampleFiles = async (dirPath, relativePrefix = '') => {
+  const results = [];
+  const localExamples = await findExampleFiles(dirPath);
+
+  for (const file of localExamples) {
+    const relativePath = relativePrefix ? `${ relativePrefix }/${ file.filename }` : file.filename;
+
+    results.push({
+      ...file,
+      relativePath,
+      displayName: relativePath,
+    });
+  }
+
+  let childDirs = [];
+
+  try {
+    const dirents = await fs.readdir(dirPath, { withFileTypes: true });
+    childDirs = dirents
+      .filter(dirent => dirent.isDirectory())
+      .map(dirent => dirent.name)
+      .filter(name => name[0] !== '_' && name[0] !== '.')
+      .filter(name => !excludedDirs.has(name));
+  } catch (err) {
+    return results;
+  }
+
+  for (const childDir of childDirs) {
+    const childPrefix = relativePrefix ? `${ relativePrefix }/${ childDir }` : childDir;
+    const childResults = await collectMineralExampleFiles(path.join(dirPath, childDir), childPrefix);
+    results.push(...childResults);
+  }
+
+  return results;
+};
+
 const sortExampleFiles = (exampleFiles) => {
   return [...exampleFiles].sort((a, b) => {
-    if (a.displayName === '_example.js') return -1;
-    if (b.displayName === '_example.js') return 1;
-    return a.displayName.localeCompare(b.displayName);
+    const aKey = a.relativePath || a.displayName;
+    const bKey = b.relativePath || b.displayName;
+
+    if (aKey === '_example.js') return -1;
+    if (bKey === '_example.js') return 1;
+
+    return aKey.localeCompare(bKey);
   });
 };
 
@@ -148,7 +208,68 @@ const normalizeTemplateArg = (templateArg) => {
   return `_example.${ templateArg }.js`;
 };
 
-const resolveTemplateFromArg = async (exampleFiles, templateArg, dir) => {
+const findWorkspaceTemplate = (exampleFiles, templateArg) => {
+  if (!templateArg) {
+    const defaultTemplate = exampleFiles.find(file => file.relativePath === '_example.js');
+    if (defaultTemplate) {
+      return defaultTemplate;
+    }
+
+    if (exampleFiles.length === 0) {
+      throw new Error('No mineral example templates found');
+    }
+
+    return exampleFiles[0];
+  }
+
+  const directMatch = exampleFiles.find(file => file.relativePath === templateArg);
+  if (directMatch) {
+    return directMatch;
+  }
+
+  const normalizedFilename = normalizeTemplateArg(templateArg);
+  const filenameMatches = exampleFiles.filter(file => file.filename === normalizedFilename);
+
+  if (templateArg.includes('/')) {
+    const [ prefix, suffix ] = templateArg.split('/');
+    const normalizedSuffix = normalizeTemplateArg(suffix);
+    const prefixedMatch = exampleFiles.find(file => file.relativePath === `${ prefix }/${ normalizedSuffix }`);
+
+    if (prefixedMatch) {
+      return prefixedMatch;
+    }
+  }
+
+  if (filenameMatches.length === 1) {
+    return filenameMatches[0];
+  }
+
+  const suffixMatches = exampleFiles.filter(file => file.relativePath.endsWith(`/${ normalizedFilename }`));
+
+  if (suffixMatches.length === 1) {
+    return suffixMatches[0];
+  }
+
+  if (suffixMatches.length > 1) {
+    throw new Error(
+      `Ambiguous template "${ templateArg }". Use one of: ${ suffixMatches.map(file => file.relativePath).join(', ') }`,
+    );
+  }
+
+  throw new Error(`Template not found: ${ templateArg }`);
+};
+
+const resolveTemplateFromArg = async ({
+  exampleFiles,
+  templateArg,
+  dir,
+  inMineral,
+}) => {
+  if (!inMineral) {
+    const selectedTemplate = findWorkspaceTemplate(exampleFiles, templateArg);
+    return selectedTemplate.fullPath;
+  }
+
   if (!templateArg) {
     const defaultInDir = exampleFiles.find(file => file.displayName === '_example.js');
     if (defaultInDir) {
@@ -184,13 +305,51 @@ const scriptFileContents = async (name, selectedTemplate) => {
   return exampleFileContents.replace(/FUNC/g, name);
 };
 
-const getDirs = async () => {
-  const dirents = await fs.readdir(apiDirectory, { withFileTypes: true });
-  return dirents
-    .filter(dirent => dirent.isDirectory())
-    .map(dirent => dirent.name)
-    .filter(name => name[0] !== '_' && name[0] !== '.')
-    .filter(name => !excludedDirs.has(name));
+const getDirs = async (targetApiDirectory) => {
+  try {
+    const dirents = await fs.readdir(targetApiDirectory, { withFileTypes: true });
+    return dirents
+      .filter(dirent => dirent.isDirectory())
+      .map(dirent => dirent.name)
+      .filter(name => name[0] !== '_' && name[0] !== '.')
+      .filter(name => !excludedDirs.has(name));
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return [];
+    }
+
+    throw err;
+  }
+};
+
+const normalizeDirArg = (dirArg) => {
+  if (!dirArg || dirArg === '.' || dirArg === 'api') {
+    return '';
+  }
+
+  return dirArg;
+};
+
+const resolveDirArg = ({ dirArg, dirs, inMineral, name }) => {
+  const normalizedDir = normalizeDirArg(dirArg);
+
+  if (dirs.includes(normalizedDir)) {
+    return normalizedDir;
+  }
+
+  if (!inMineral && dirs.length === 0 && normalizedDir === '') {
+    return '';
+  }
+
+  if (!inMineral && !dirArg && name) {
+    return '';
+  }
+
+  if (inMineral && !dirArg) {
+    return null;
+  }
+
+  return normalizedDir;
 };
 
 const buildFuncName = (dir, name) => {
@@ -201,6 +360,11 @@ const buildFuncName = (dir, name) => {
 };
 
 const selectDirInteractive = async (dirs) => {
+  if (dirs.length === 0) {
+    console.log('\nCreating in api/');
+    return '';
+  }
+
   const dirIndex = await askQuestion(`Where does your new function live? \n${
     dirs.map((dir, index) => {
       return `[${ index + 1 }] ${ dir }`;
@@ -216,33 +380,54 @@ const selectDirInteractive = async (dirs) => {
   return dir;
 };
 
-const selectTemplateInteractive = async (exampleFiles, dir) => {
+const getExampleFilesForContext = async ({ context, dir }) => {
+  if (context.inMineral) {
+    return sortExampleFiles(await findExampleFiles(path.join(context.apiDirectory, dir)));
+  }
+
+  return sortExampleFiles(await collectMineralExampleFiles(context.mineralApiDirectory));
+};
+
+const formatTemplateLabel = (file, { inMineral }) => {
+  if (inMineral) {
+    return file.displayName;
+  }
+
+  return file.relativePath;
+};
+
+const selectTemplateInteractive = async ({ exampleFiles, dir, inMineral }) => {
   if (exampleFiles.length === 0) {
     await fs.access(rootExampleJsPath);
-    console.log('\nUsing template: api/_example.js');
+    console.log('\nUsing template: _example.js');
     return rootExampleJsPath;
   }
 
   if (exampleFiles.length === 1) {
-    console.log(`\nUsing template: ${ exampleFiles[0].displayName }`);
+    console.log(`\nUsing template: ${ formatTemplateLabel(exampleFiles[0], { inMineral }) }`);
     return exampleFiles[0].fullPath;
   }
 
-  console.log(`\nFound ${ exampleFiles.length } template(s) in api/${ dir }:`);
+  const templateScope = inMineral ? `in api/${ dir }` : 'from mineral';
+  console.log(`\nFound ${ exampleFiles.length } template(s) ${ templateScope }:`);
+
   const templateIndex = await askQuestion(`Which template would you like to use? (press enter for _example.js) \n${
     exampleFiles.map((file, index) => {
-      return `[${ index + 1 }] ${ file.displayName }`;
+      return `[${ index + 1 }] ${ formatTemplateLabel(file, { inMineral }) }`;
     }).join('\n')
   }\n`);
 
   if (!templateIndex || templateIndex.trim() === '') {
-    const defaultFile = exampleFiles.find(file => file.displayName === '_example.js');
+    const defaultFile = exampleFiles.find(file => (
+      file.relativePath === '_example.js' || file.displayName === '_example.js'
+    ));
+
     if (defaultFile) {
       return defaultFile.fullPath;
     }
 
     await fs.access(rootExampleJsPath);
-    console.log('\nUsing template: api/_example.js');
+    console.log('\nUsing template: _example.js');
     return rootExampleJsPath;
   }
 
@@ -266,6 +451,7 @@ const selectNameInteractive = async (dir) => {
 };
 
 const writeNewFunction = async ({
+  apiDirectory,
   dir,
   name,
   selectedTemplate,
@@ -283,6 +469,8 @@ const writeNewFunction = async ({
     }
   }
 
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+
   const script = await scriptFileContents(funcName, selectedTemplate);
   await fs.writeFile(outputPath, script);
 
@@ -296,38 +484,55 @@ const writeNewFunction = async ({
 };
 
 const createNewFunction = async () => {
+  const context = getContext();
   const cliArgs = parseCliArgs(process.argv.slice(2));
 
   if (cliArgs.help) {
-    printHelp();
+    printHelp(context);
     return;
   }
 
-  const dirs = await getDirs();
+  const dirs = await getDirs(context.apiDirectory);
   const nonInteractive = Boolean(cliArgs.dir || cliArgs.name);
 
   if (nonInteractive) {
-    if (!cliArgs.dir || !cliArgs.name) {
-      console.error('Non-interactive mode requires both --dir and --name.');
-      printHelp();
+    if (!cliArgs.name) {
+      console.error('Non-interactive mode requires --name.');
+      printHelp(context);
       process.exitCode = 1;
       return;
     }
 
-    if (!dirs.includes(cliArgs.dir)) {
-      console.error(`Invalid --dir "${ cliArgs.dir }". Available: ${ dirs.join(', ') }`);
+    const dir = resolveDirArg({
+      dirArg: cliArgs.dir,
+      dirs,
+      inMineral: context.inMineral,
+      name: cliArgs.name,
+    });
+
+    if (dir === null || (!dirs.includes(dir) && !(dir === '' && dirs.length === 0 && !context.inMineral))) {
+      const availableDirs = dirs.length ? dirs.join(', ') : 'api/ (flat)';
+      console.error(`Invalid --dir "${ cliArgs.dir }". Available: ${ availableDirs }`);
       process.exitCode = 1;
       return;
     }
 
-    const exampleFiles = sortExampleFiles(await findExampleFiles(path.join(apiDirectory, cliArgs.dir)));
+    const exampleFiles = await getExampleFilesForContext({ context, dir });
 
     try {
-      const selectedTemplate = await resolveTemplateFromArg(exampleFiles, cliArgs.template, cliArgs.dir);
-      console.log(`Using template: ${ path.relative(path.join(__dirname, '..'), selectedTemplate) }`);
+      const selectedTemplate = await resolveTemplateFromArg({
+        exampleFiles,
+        templateArg: cliArgs.template,
+        dir,
+        inMineral: context.inMineral,
+      });
+
+      const templateLabel = path.relative(mineralRoot, selectedTemplate);
+      console.log(`Using template: ${ templateLabel }`);
 
       await writeNewFunction({
-        dir: cliArgs.dir,
+        apiDirectory: context.apiDirectory,
+        dir,
         name: cliArgs.name,
         selectedTemplate,
         shouldAutoCommit: process.env.AUTO_COMMIT_STUBS === 'true' || cliArgs.commit,
@@ -342,11 +547,16 @@ const createNewFunction = async () => {
 
   try {
     const dir = await selectDirInteractive(dirs);
-    const exampleFiles = sortExampleFiles(await findExampleFiles(path.join(apiDirectory, dir)));
-    const selectedTemplate = await selectTemplateInteractive(exampleFiles, dir);
+    const exampleFiles = await getExampleFilesForContext({ context, dir });
+    const selectedTemplate = await selectTemplateInteractive({
+      exampleFiles,
+      dir,
+      inMineral: context.inMineral,
+    });
     const name = await selectNameInteractive(dir);
 
     await writeNewFunction({
+      apiDirectory: context.apiDirectory,
       dir,
       name,
       selectedTemplate,

package/bin/mineral.js

@@ -3,7 +3,7 @@
 const command = process.argv[2];
 
 if (command === 'host') {
-  const { deployFromHostingYml } = require('../_deploy_scripts/deployFromHostingYml');
+  const { deployFromHostingYml } = require('../hosting/deployFromHostingYml');
   deployFromHostingYml().catch((error) => {
     console.error(error);
     process.exit(1);

package/hosting/.hosting.yml.sample

@@ -0,0 +1,23 @@
+google_cloud_info:
+  project:
+  region:
+
+# TODO: consider credsPayload in google_cloud_info instead of workspace .creds.yml
+
+functions:
+  exampleFunction:
+    max_instances: 1
+    timeout: 300s
+    wrappers:
+      - requireHostedApiKey
+      - allowCrossOriginCallsAndHandleOptions
+    # entry_point: otherHandlerName
+
+  pokemonPokeballThrow:
+    max_instances: 1
+    wrappers:
+      - checkTrainer
+
+groups:
+  example_group:
+    - exampleFunction

package/hosting/copyCredsToEnv.js

@@ -0,0 +1,48 @@
+const fs = require('fs');
+const yaml = require('yaml');
+
+const copyCredsToEnv = (workspace) => {
+  const normalizedWorkspace = workspace.replace(/\/$/, '');
+  const credsPath = `${ normalizedWorkspace }/.creds.yml`;
+  const envPath = `${ normalizedWorkspace }/.env`;
+
+  if (!fs.existsSync(credsPath)) {
+    throw new Error(`Missing .creds.yml in workspace: ${ workspace }`);
+  }
+
+  const credsText = fs.readFileSync(credsPath, 'utf8');
+  const credsFromYml = yaml.parse(credsText);
+  const newCredsLine = `CREDS=${ JSON.stringify(credsFromYml) }`;
+
+  let envFileContents = '';
+  if (fs.existsSync(envPath)) {
+    envFileContents = fs.readFileSync(envPath, 'utf8');
+  }
+
+  if (!envFileContents) {
+    fs.writeFileSync(envPath, `${ newCredsLine }\n`);
+    return;
+  }
+
+  if (/^CREDS=/m.test(envFileContents)) {
+    const updatedFileContents = envFileContents.replace(/^CREDS=.*$/m, newCredsLine);
+
+    if (updatedFileContents !== envFileContents) {
+      fs.writeFileSync(envPath, updatedFileContents);
+    }
+
+    return;
+  }
+
+  fs.appendFileSync(envPath, `\n\n${ newCredsLine }\n`);
+};
+
+if (require.main === module) {
+  const workspace = process.argv[2] || process.cwd();
+  copyCredsToEnv(workspace);
+  console.log('Copied CREDS from .creds.yml to .env');
+}
+
+module.exports = {
+  copyCredsToEnv,
+};

package/{_deploy_scripts → hosting}/deployFromHostingYml.js

@@ -1,14 +1,13 @@
 const readline = require('readline');
-const { toAbsolutePath, setWorkspace } = require('../api/workspace');
+const { toAbsolutePath, setWorkspace, loadWorkspaceEnv } = require('../api/workspace');
 const { getApiDirs, readCliFlag } = require('../cli');
 const { loadHandlers } = require('../server');
 const {
   readHostingYml,
-  getCredsJsonForDeploy,
-  validateEnvVarsForDeploy,
+  ensureWorkspaceEnvForDeploy,
   resolveHostedHandlersForDeploy,
   functionUsesWrapper,
-} = require('../hosting.utils');
+} = require('./hosting.utils');
 const { writeHostedJs } = require('./generateHosted');
 const { execCommand } = require('./execCommand');
 const { formatSetEnvVarsForGcloud, shellQuoteSingle } = require('./setEnvVarsGcloud');
@@ -91,8 +90,6 @@ const deployFunction = async ({
   functionConfig,
   googleCloudInfo,
   workspace,
-  credsJson,
-  deployEnvVars = {},
 }) => {
   const config = {
     ...googleCloudInfo,
@@ -107,20 +104,16 @@ const deployFunction = async ({
     allow_unauthenticated: allowUnauthenticated = true,
     gen2 = true,
     set_env_vars: extraSetEnvVars,
-    entry_point: entryPoint,
+    entry_point,
     schedules,
     groups,
     wrappers,
     source,
+    env,
     ...gcloudArgs
   } = config;
 
-  const resolvedEntryPoint = entryPoint || functionName;
-  const envParts = [
-    'HOSTED=true',
-    `CREDS=${ credsJson }`,
-    ...Object.entries(deployEnvVars).map(([envName, envValue]) => `${ envName }=${ envValue }`),
-  ];
+  const envParts = ['HOSTED=true'];
 
   if (extraSetEnvVars) {
     envParts.push(extraSetEnvVars);
@@ -134,7 +127,7 @@ const deployFunction = async ({
     `--project ${ project }`,
     `--region ${ region }`,
     `--source ${ shellQuoteSingle(workspace) }`,
-    `--entry-point ${ resolvedEntryPoint }`,
+    `--entry-point ${ functionName }`,
     `--trigger-${ trigger }`,
     `--runtime ${ runtime }`,
     `--set-env-vars ${ shellQuoteSingle(setEnvVarsForGcloud) }`,
@@ -157,7 +150,7 @@ const deployFunction = async ({
   }
 
   const requiresHostedApiKey = functionUsesWrapper(functionConfig, 'requireHostedApiKey');
-  const hostedApiKey = deployEnvVars.HOSTED_API_KEY;
+  const hostedApiKey = process.env.HOSTED_API_KEY;
 
   for (const schedule of schedules) {
     const {
@@ -203,17 +196,18 @@ const deployFunction = async ({
 const deployFromHostingYml = async (options = {}) => {
   const config = getHostConfig(options);
   setWorkspace(config.workspace);
+  ensureWorkspaceEnvForDeploy(config.workspace);
+  loadWorkspaceEnv();
 
   const hostingConfig = readHostingYml(config.workspace);
   const {
     google_cloud_info: googleCloudInfo,
-    env: envNames = [],
     functions = {},
     groups = {},
   } = hostingConfig;
 
   if (!googleCloudInfo?.project || !googleCloudInfo?.region) {
-    throw new Error('.hosting.yml requires google_cloud_info.project and google_cloud_info.region');
+    throw new Error('hosting/.hosting.yml requires google_cloud_info.project and google_cloud_info.region');
   }
 
   const handlers = loadHandlers({
@@ -227,19 +221,16 @@ const deployFromHostingYml = async (options = {}) => {
     handlersByName,
   });
 
-  const deployEnvVars = validateEnvVarsForDeploy(config.workspace, envNames);
-
   writeHostedJs({
     workspace: config.workspace,
     hostedHandlers,
   });
 
-  const credsJson = getCredsJsonForDeploy(config.workspace);
   const deployArgs = getDeployArgs();
 
   const deployOne = async (functionName) => {
     if (!functions[functionName]) {
-      console.log(`Function ${ functionName } not found in .hosting.yml, skipping`);
+      console.log(`Function ${ functionName } not found in hosting/.hosting.yml, skipping`);
       return;
     }
 
@@ -248,15 +239,13 @@ const deployFromHostingYml = async (options = {}) => {
       functionConfig: functions[functionName],
       googleCloudInfo,
       workspace: config.workspace,
-      credsJson,
-      deployEnvVars,
     });
   };
 
   if (deployArgs.includes('group')) {
     const groupNames = Object.keys(groups);
     if (!groupNames.length) {
-      console.log('No groups defined in .hosting.yml');
+      console.log('No groups defined in hosting/.hosting.yml');
       return;
     }
 
@@ -270,7 +259,7 @@ const deployFromHostingYml = async (options = {}) => {
   if (deployArgs.includes('function')) {
     const functionNames = Object.keys(functions);
     if (!functionNames.length) {
-      console.log('No functions defined in .hosting.yml');
+      console.log('No functions defined in hosting/.hosting.yml');
       return;
     }
 
@@ -305,14 +294,3 @@ module.exports = {
   deployFromHostingYml,
   getHostConfig,
 };
-
-/*
-  Deploy everything:
-  mineral host all
-
-  Deploy a single function:
-  mineral host function
-
-  Deploy a group of functions:
-  mineral host group
-*/

package/hosting/generateHosted.js

@@ -0,0 +1,60 @@
+const fs = require('fs');
+const { wrapHostedFunction } = require('./hosting.utils');
+
+const formatResolvedWrapperForHostedJs = ({ modulePath, wrapperName }) => (
+  `require('${ modulePath }').${ wrapperName }`
+);
+
+const formatWrappersArg = (resolvedWrappers = []) => {
+  if (!resolvedWrappers.length) {
+    return '';
+  }
+
+  const wrapperLines = resolvedWrappers.map((resolvedWrapper) => (
+    `    ${ formatResolvedWrapperForHostedJs(resolvedWrapper) },`
+  ));
+  return `, [\n${ wrapperLines.join('\n') }\n  ]`;
+};
+
+const generateHostedJs = ({
+  hostedHandlers,
+}) => {
+  const exportLines = [];
+
+  for (const hostedHandler of hostedHandlers) {
+    const { hostedName, handlerName, resolvedWrappers = [], requirePath } = hostedHandler;
+    const wrappersArg = formatWrappersArg(resolvedWrappers);
+
+    exportLines.push(
+      `  ${ hostedName }: wrapHostedFunction(() => require('${ requirePath }'), '${ handlerName }'${ wrappersArg }),`,
+    );
+  }
+
+  return `// Generated by mineral — do not edit
+const { wrapHostedFunction } = require('@foxtware/mineral/hosting/hosting.utils');
+
+module.exports = {
+${ exportLines.join('\n') }
+};
+`;
+};
+
+const writeHostedJs = ({
+  workspace,
+  hostedHandlers,
+}) => {
+  const { getHostingDir } = require('./hosting.utils');
+  const hostedPath = `${ getHostingDir(workspace) }/hosted.js`;
+  const content = generateHostedJs({
+    hostedHandlers,
+  });
+
+  fs.mkdirSync(getHostingDir(workspace), { recursive: true });
+  fs.writeFileSync(hostedPath, content);
+  return hostedPath;
+};
+
+module.exports = {
+  generateHostedJs,
+  writeHostedJs,
+};

package/hosting/handlerPaths.js

@@ -0,0 +1,23 @@
+const MINERAL_ROOT = `${ __dirname }/..`;
+const MINERAL_API_DIR = `${ MINERAL_ROOT }/api`;
+
+const getRequirePathForHandler = (handler, workspace) => {
+  const normalizedWorkspace = workspace.replace(/\/$/, '');
+  const { filePath } = handler;
+
+  if (filePath.startsWith(`${ normalizedWorkspace }/`)) {
+    return `./${ filePath.slice(normalizedWorkspace.length + 1) }`;
+  }
+
+  if (filePath.startsWith(`${ MINERAL_API_DIR }/`)) {
+    const relativePath = filePath.slice(MINERAL_API_DIR.length + 1);
+    return `@foxtware/mineral/api/${ relativePath }`;
+  }
+
+  throw new Error(`Handler "${ handler.routeName }" is not in workspace or mineral api dirs`);
+};
+
+module.exports = {
+  MINERAL_API_DIR,
+  getRequirePathForHandler,
+};

package/{hosting.utils.js → hosting/hosting.utils.js}

@@ -1,6 +1,9 @@
 const fs = require('fs');
+const path = require('path');
+const dotenv = require('dotenv');
 const { createRequire } = require('module');
 const yaml = require('yaml');
+const { getRequirePathForHandler } = require('./handlerPaths');
 const {
   respondJson,
   errorToReadable,
@@ -9,63 +12,71 @@ const {
   funcApi,
   wrapFunction,
   statusCodeFromResult,
-} = require('./server.utils');
+} = require('../server.utils');
 
-const parseWrapperRef = (wrapperRef) => {
-  const hashIndex = wrapperRef.lastIndexOf('#');
+dotenv.config({
+  path: path.join(process.cwd(), '.env'),
+});
 
-  if (hashIndex === -1) {
-    throw new Error(`Invalid wrapper ref (expected path#export): ${ wrapperRef }`);
-  }
+const MINERAL_WRAPPERS_MODULE = '@foxtware/mineral/hosting/wrappers.js';
+const WORKSPACE_WRAPPERS_MODULE = './hosting/wrappers.js';
 
-  return {
-    modulePath: wrapperRef.slice(0, hashIndex),
-    exportName: wrapperRef.slice(hashIndex + 1),
-  };
-};
+const getHostingDir = (workspace) => `${ workspace.replace(/\/$/, '') }/hosting`;
 
-const validateWrapperRef = (wrapperRef, workspaceRequire) => {
-  const { modulePath, exportName } = parseWrapperRef(wrapperRef);
-  workspaceRequire.resolve(modulePath);
-  const moduleExports = workspaceRequire(modulePath);
+const getMineralWrappers = (workspaceRequire) => {
+  try {
+    return workspaceRequire(MINERAL_WRAPPERS_MODULE);
+  } catch (error) {
+    if (error.code !== 'MODULE_NOT_FOUND') {
+      throw error;
+    }
 
-  if (typeof moduleExports[exportName] !== 'function') {
-    throw new Error(`Wrapper export not found: ${ exportName } in ${ modulePath }`);
+    return require('./wrappers.js');
   }
 };
 
-const wrapperExportName = (wrapperRef) => (
-  parseWrapperRef(wrapperRef).exportName
-);
-
-const getHostedEntries = (functions = {}) => {
-  const byEntryPoint = new Map();
+const resolveWrapperName = (wrapperName, workspaceRequire) => {
+  if (typeof wrapperName !== 'string' || !wrapperName.trim()) {
+    throw new Error(`Invalid wrapper name: ${ wrapperName }`);
+  }
 
-  for (const [functionName, functionConfig] of Object.entries(functions)) {
-    const entryPoint = functionConfig.entry_point || functionConfig.entryPoint || functionName;
-    const existing = byEntryPoint.get(entryPoint) || {
-      entryPoint,
-      wrappers: [],
+  const mineralWrappers = getMineralWrappers(workspaceRequire);
+  if (typeof mineralWrappers[wrapperName] === 'function') {
+    return {
+      wrapperName,
+      modulePath: MINERAL_WRAPPERS_MODULE,
     };
+  }
 
-    if (Array.isArray(functionConfig.wrappers)) {
-      existing.wrappers.push(...functionConfig.wrappers);
-      existing.wrappers = [...new Set(existing.wrappers)];
+  try {
+    const workspaceWrappers = workspaceRequire(WORKSPACE_WRAPPERS_MODULE);
+    if (typeof workspaceWrappers[wrapperName] === 'function') {
+      return {
+        wrapperName,
+        modulePath: WORKSPACE_WRAPPERS_MODULE,
+      };
     }
-
-    if (functionConfig.source) {
-      existing.source = functionConfig.source;
+  } catch (error) {
+    if (error.code !== 'MODULE_NOT_FOUND') {
+      throw error;
     }
-
-    byEntryPoint.set(entryPoint, existing);
   }
 
-  return [...byEntryPoint.values()];
+  throw new Error(
+    `Wrapper "${ wrapperName }" not found in ${ MINERAL_WRAPPERS_MODULE } or ${ WORKSPACE_WRAPPERS_MODULE }`,
+  );
 };
 
-const functionUsesWrapper = (functionConfig = {}, exportName) => (
-  Array.isArray(functionConfig.wrappers)
-  && functionConfig.wrappers.some((wrapperRef) => wrapperExportName(wrapperRef) === exportName)
+const getHostedEntries = (functions = {}) => (
+  Object.entries(functions).map(([hostedName, functionConfig = {}]) => ({
+    hostedName,
+    handlerName: functionConfig.entry_point || functionConfig.entryPoint || hostedName,
+    wrappers: Array.isArray(functionConfig.wrappers) ? functionConfig.wrappers : [],
+  }))
+);
+
+const functionUsesWrapper = (functionConfig = {}, wrapperName) => (
+  Array.isArray(functionConfig.wrappers) && functionConfig.wrappers.includes(wrapperName)
 );
 
 const getFuncApiConfig = ({
@@ -153,17 +164,17 @@ const wrapHostedFunction = (loader, exportName, wrappers = []) => {
 };
 
 const readHostingYml = (workspace) => {
-  const hostingPath = `${ workspace }/.hosting.yml`;
+  const hostingPath = `${ getHostingDir(workspace) }/.hosting.yml`;
 
   if (!fs.existsSync(hostingPath)) {
-    throw new Error(`Missing .hosting.yml in workspace: ${ workspace }`);
+    throw new Error(`Missing hosting/.hosting.yml in workspace: ${ workspace }`);
   }
 
   const hostingText = fs.readFileSync(hostingPath, 'utf8');
   const hostingConfig = yaml.parse(hostingText);
 
   if (!hostingConfig || typeof hostingConfig !== 'object' || Array.isArray(hostingConfig)) {
-    throw new Error('Invalid .hosting.yml');
+    throw new Error('Invalid hosting/.hosting.yml');
   }
 
   return hostingConfig;
@@ -180,33 +191,15 @@ const getCredsJsonForDeploy = (workspace) => {
   return JSON.stringify(yaml.parse(credsText));
 };
 
-const getEnvValueForDeploy = (workspace, envName) => {
-  const envPath = `${ workspace }/.env`;
+const ensureWorkspaceEnvForDeploy = (workspace) => {
+  const { copyCredsToEnv } = require('./copyCredsToEnv');
+  const envPath = `${ workspace.replace(/\/$/, '') }/.env`;
 
-  if (!fs.existsSync(envPath)) {
-    return '';
-  }
-
-  const envText = fs.readFileSync(envPath, 'utf8');
-  const match = envText.match(new RegExp(`^${ envName }=(.*)$`, 'm'));
-  return match ? match[1].trim() : '';
-};
-
-const getEnvVarsForDeploy = (workspace, envNames = []) => (
-  Object.fromEntries(
-    envNames.map((envName) => [envName, getEnvValueForDeploy(workspace, envName)]),
-  )
-);
-
-const validateEnvVarsForDeploy = (workspace, envNames = []) => {
-  const envVars = getEnvVarsForDeploy(workspace, envNames);
-  const missing = envNames.filter((envName) => !envVars[envName]);
+  copyCredsToEnv(workspace);
 
-  if (missing.length) {
-    throw new Error(`Missing required .env values: ${ missing.join(', ') }`);
+  if (!fs.existsSync(envPath)) {
+    throw new Error(`Missing .env in workspace: ${ workspace }`);
   }
-
-  return envVars;
 };
 
 const resolveHostedHandlersForDeploy = ({
@@ -218,35 +211,23 @@ const resolveHostedHandlersForDeploy = ({
   const hostedEntries = getHostedEntries(functions);
 
   return hostedEntries.map((hostedEntry) => {
-    const { entryPoint, source, wrappers = [] } = hostedEntry;
+    const { handlerName, wrappers = [] } = hostedEntry;
 
-    for (const wrapperRef of wrappers) {
-      validateWrapperRef(wrapperRef, workspaceRequire);
-    }
-
-    if (source) {
-      workspaceRequire.resolve(source);
-      const moduleExports = workspaceRequire(source);
-      if (typeof moduleExports[entryPoint] !== 'function') {
-        throw new Error(`Hosted export not found: ${ entryPoint } in ${ source }`);
-      }
-
-      return {
-        ...hostedEntry,
-        requirePath: source,
-      };
-    }
+    const resolvedWrappers = wrappers.map((wrapperName) => (
+      resolveWrapperName(wrapperName, workspaceRequire)
+    ));
 
-    const handler = handlersByName.get(entryPoint);
+    const handler = handlersByName.get(handlerName);
     if (!handler) {
       throw new Error(
-        `entry_point "${ entryPoint }" not found in workspace handlers — add api/ handler or source in .hosting.yml`,
+        `Function "${ handlerName }" not found — add a handler in workspace api dirs or mineral api`,
       );
     }
 
     return {
       ...hostedEntry,
-      requirePath: `./${ handler.filePath.slice(workspace.length + 1) }`,
+      resolvedWrappers,
+      requirePath: getRequirePathForHandler(handler, workspace),
     };
   });
 };
@@ -254,15 +235,16 @@ const resolveHostedHandlersForDeploy = ({
 // TODO: support credsPayload in google_cloud_info instead of full workspace .creds.yml
 
 module.exports = {
-  parseWrapperRef,
+  getHostingDir,
+  resolveWrapperName,
   getFuncApiConfig,
   wrapHostedFunction,
   readHostingYml,
   getCredsJsonForDeploy,
-  getEnvVarsForDeploy,
-  validateEnvVarsForDeploy,
+  ensureWorkspaceEnvForDeploy,
   getHostedEntries,
   resolveHostedHandlersForDeploy,
   functionUsesWrapper,
-  wrapperExportName,
+  MINERAL_WRAPPERS_MODULE,
+  WORKSPACE_WRAPPERS_MODULE,
 };

package/hosting/wrappers.js

@@ -0,0 +1,37 @@
+const { HOSTED } = require('../api/constants');
+
+const requireHostedApiKey = async (req) => {
+  if (!HOSTED) {
+    return;
+  }
+
+  if (req.headers['x-api-key'] !== process.env.HOSTED_API_KEY) {
+    return {
+      ok: false,
+      error: {
+        code: 'UNAUTHORIZED',
+        message: 'Unauthorized',
+        statusCode: 401,
+      },
+    };
+  }
+};
+
+const allowCrossOriginCallsAndHandleOptions = async (req, res) => {
+  const { origin } = req.headers;
+
+  res.setHeader('Access-Control-Allow-Origin', origin || '*');
+  res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+  res.setHeader('Access-Control-Allow-Headers', 'Content-Type, x-api-key, x-wf-token, x-wf-value');
+
+  if (req.method === 'OPTIONS') {
+    res.writeHead(204);
+    res.end();
+    return { handled: true };
+  }
+};
+
+module.exports = {
+  requireHostedApiKey,
+  allowCrossOriginCallsAndHandleOptions,
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@foxtware/mineral",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "bin": {
     "mineral": "bin/mineral.js"
   },
@@ -9,7 +9,7 @@
     "access": "public"
   },
   "scripts": {
-    "creds_to_env": "node _build_scripts/copyCredsToEnv.js",
+    "creds_to_env": "node hosting/copyCredsToEnv.js",
     "dev": "node --watch --watch-path=./api --watch-path=./server.js --watch-path=./server.utils.js --watch-path=./.creds.yml server.js",
     "new": "node _build_scripts/createNewFunction.js",
     "serve": "PORT=8100 node server.js",

package/server.js

@@ -3,9 +3,8 @@ const http = require('http');
 const { respondJson, errorToReadable, getRequestBody, argsFromBody, funcApi, statusCodeFromResult } = require('./server.utils');
 const { getWorkspace, setWorkspace, loadWorkspaceEnv, toAbsolutePath } = require('./api/workspace');
 const { getApiDirs, readCliFlag } = require('./cli');
-const { getFuncApiConfig } = require('./hosting.utils');
-
-const MINERAL_API_DIR = `${ __dirname }/api`;
+const { getFuncApiConfig } = require('./hosting/hosting.utils');
+const { MINERAL_API_DIR } = require('./hosting/handlerPaths');
 
 const getConfig = (options = {}) => ({
   port: Number(options.port ?? process.env.PORT ?? 8000),
@@ -69,7 +68,6 @@ const listJsFiles = (directory) => {
 const directoriesToScan = ({
   workspace,
   api_dirs,
-  host_mode = false,
 }) => {
   const extraDirs = api_dirs.map((dir) => (
     dir.startsWith('/')
@@ -77,10 +75,6 @@ const directoriesToScan = ({
       : `${ workspace }/${ dir }`
   ));
 
-  if (host_mode && api_dirs.length) {
-    return extraDirs;
-  }
-
   return [MINERAL_API_DIR, ...extraDirs];
 };
 
@@ -225,6 +219,7 @@ const startServer = (options = {}) => {
 module.exports = {
   startServer,
   loadHandlers,
+  MINERAL_API_DIR,
   get server() {
     if (!server) {
       startServer();

package/server.utils.js

@@ -1,6 +1,5 @@
 const { logDeep } = require('./api/utils');
 const { StringDecoder } = require('string_decoder');
-const { HOSTED } = require('./api/constants');
 
 const respondJson = (res, statusCode, payload) => {
   logDeep(payload);
@@ -132,37 +131,6 @@ const wrapFunction = (func, wrappers = []) => async (req, res, ...rest) => {
   return func(req, res, ...rest);
 };
 
-const requireHostedApiKey = async (req) => {
-  if (!HOSTED) {
-    return;
-  }
-
-  if (req.headers['x-api-key'] !== process.env.HOSTED_API_KEY) {
-    return {
-      ok: false,
-      error: {
-        code: 'UNAUTHORIZED',
-        message: 'Unauthorized',
-        statusCode: 401,
-      },
-    };
-  }
-};
-
-const allowCrossOriginCallsAndHandleOptions = async (req, res) => {
-  const { origin } = req.headers;
-
-  res.setHeader('Access-Control-Allow-Origin', origin || '*');
-  res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
-  res.setHeader('Access-Control-Allow-Headers', 'Content-Type, x-api-key, x-wf-token, x-wf-value');
-
-  if (req.method === 'OPTIONS') {
-    res.writeHead(204);
-    res.end();
-    return { handled: true };
-  }
-};
-
 const statusCodeFromResult = (result) => {
   if (result?.ok === false) {
     return result?.error?.statusCode ?? 400;
@@ -280,8 +248,6 @@ module.exports = {
   getRequestBody,
   argsFromBody,
   wrapFunction,
-  requireHostedApiKey,
-  allowCrossOriginCallsAndHandleOptions,
   statusCodeFromResult,
   funcApi,
 };

package/.hosting.yml.sample

@@ -1,27 +0,0 @@
-google_cloud_info:
-  project:
-  region:
-
-# TODO: consider credsPayload in google_cloud_info instead of workspace .creds.yml
-
-env:
-  - HOSTED_API_KEY
-
-functions:
-  exampleFunction:
-    max_instances: 1
-    timeout: 300s
-    wrappers:
-      - '@foxtware/mineral/server.utils#requireHostedApiKey'
-      - '@foxtware/mineral/server.utils#allowCrossOriginCallsAndHandleOptions'
-    # entry_point: otherExportName
-
-  packagedFunction:
-    source: '@foxtware/mineral/api/pokemon/pokemonPokeballThrow.js'
-    max_instances: 1
-    wrappers:
-      - '@foxtware/mineral/server.utils#checkTrainer'
-
-groups:
-  example_group:
-    - exampleFunction

package/_build_scripts/copyCredsToEnv.js

@@ -1,45 +0,0 @@
-const fs = require('fs').promises;
-const path = require('path');
-const yaml = require('yaml');
-
-const mineralRoot = path.join(__dirname, '..');
-const credsYmlPath = path.join(mineralRoot, '.creds.yml');
-const envPath = path.join(mineralRoot, '.env');
-
-(async () => {
-  const credsText = await fs.readFile(credsYmlPath, 'utf8');
-  const credsFromYml = yaml.parse(credsText);
-  const credsJsonString = JSON.stringify(credsFromYml);
-  const newCredsLine = `CREDS=${ credsJsonString }`;
-
-  let envFileContents = '';
-  try {
-    envFileContents = await fs.readFile(envPath, 'utf8');
-  } catch (err) {
-    if (err.code !== 'ENOENT') {
-      throw err;
-    }
-  }
-
-  if (!envFileContents) {
-    await fs.writeFile(envPath, `${ newCredsLine }\n`);
-    console.log('Created .env with CREDS');
-    return;
-  }
-
-  if (/^CREDS=/m.test(envFileContents)) {
-    const updatedFileContents = envFileContents.replace(/^CREDS=.*$/m, newCredsLine);
-
-    if (updatedFileContents === envFileContents) {
-      console.log('CREDS already up to date');
-      return;
-    }
-
-    await fs.writeFile(envPath, updatedFileContents);
-    console.log('Updated CREDS in .env');
-    return;
-  }
-
-  await fs.appendFile(envPath, `\n\n${ newCredsLine }\n`);
-  console.log('Appended CREDS to .env');
-})();

package/_deploy_scripts/generateHosted.js

@@ -1,57 +0,0 @@
-const fs = require('fs');
-const { wrapHostedFunction, parseWrapperRef } = require('../hosting.utils');
-
-const formatWrapperRefForHostedJs = (wrapperRef) => {
-  const { modulePath, exportName } = parseWrapperRef(wrapperRef);
-  return `require('${ modulePath }').${ exportName }`;
-};
-
-const formatWrappersArg = (wrappers = []) => {
-  if (!wrappers.length) {
-    return '';
-  }
-
-  const wrapperLines = wrappers.map((wrapperRef) => `    ${ formatWrapperRefForHostedJs(wrapperRef) },`);
-  return `, [\n${ wrapperLines.join('\n') }\n  ]`;
-};
-
-const generateHostedJs = ({
-  hostedHandlers,
-}) => {
-  const exportLines = [];
-
-  for (const hostedHandler of hostedHandlers) {
-    const { entryPoint, wrappers = [], requirePath } = hostedHandler;
-    const wrappersArg = formatWrappersArg(wrappers);
-
-    exportLines.push(
-      `  ${ entryPoint }: wrapHostedFunction(() => require('${ requirePath }'), '${ entryPoint }'${ wrappersArg }),`,
-    );
-  }
-
-  return `// Generated by mineral — do not edit
-const { wrapHostedFunction } = require('@foxtware/mineral/hosting.utils');
-
-module.exports = {
-${ exportLines.join('\n') }
-};
-`;
-};
-
-const writeHostedJs = ({
-  workspace,
-  hostedHandlers,
-}) => {
-  const hostedPath = `${ workspace.replace(/\/$/, '') }/hosted.js`;
-  const content = generateHostedJs({
-    hostedHandlers,
-  });
-
-  fs.writeFileSync(hostedPath, content);
-  return hostedPath;
-};
-
-module.exports = {
-  generateHostedJs,
-  writeHostedJs,
-};