@foxtware/mineral 0.1.5 → 0.1.7

package/api/slack/slack.utils.js

@@ -0,0 +1,46 @@
+const { SLACK_API_BASE_URL } = require('../slack/slack.constants');
+const {
+  FetchClient,
+  Chain,
+  appendUrlToBase,
+  fetchClientCommonSteps,
+} = require('../utils');
+
+const addUrlAndAuthHeaders = async (state) => {
+  const { requestPayload, context } = state;
+  const { creds } = context;
+  const {
+    BOT_TOKEN,
+  } = creds;
+
+  return {
+    requestPayload: {
+      ...requestPayload,
+      method: requestPayload.method || 'post',
+      url: appendUrlToBase(SLACK_API_BASE_URL, requestPayload.url),
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${ BOT_TOKEN }`,
+        ...requestPayload.headers,
+      },
+    },
+  };
+};
+
+
+const slackClientRequestPreparer = new Chain([
+  addUrlAndAuthHeaders,
+]);
+
+const slackClientResponseInterpreter = new Chain([
+  fetchClientCommonSteps.exitEarlyOnNotOk,
+]);
+
+const slackClient = new FetchClient({
+  requestPreparer: slackClientRequestPreparer,
+  responseInterpreter: slackClientResponseInterpreter,
+});
+
+module.exports = {
+  slackClient,
+};

package/api/slack/slackMessagePost.js

@@ -0,0 +1,85 @@
+// https://docs.slack.dev/reference/methods/chat.postmessage
+
+const { credsFromPayload, objHasAny, ArgsWarden } = require('../utils');
+const { credsValidator } = require('../validators');
+const { slackClient } = require('../slack/slack.utils');
+
+const channelIdentifierValidator = (channelIdentifier) => {
+  return objHasAny(channelIdentifier, ['channelName', 'channelId']);
+};
+
+const messagePayloadValidator = (messagePayload) => {
+  return objHasAny(messagePayload, ['text', 'blocks', 'markdownText']);
+};
+
+const argsWarden = new ArgsWarden([
+  ['credsPayload', credsValidator],
+  ['channelIdentifier', channelIdentifierValidator],
+  ['messagePayload', messagePayloadValidator],
+]);
+
+const slackMessagePost = async (
+  credsPayload,
+  channelIdentifier,
+  messagePayload,
+  {
+    inspect = false,
+  } = {},
+) => {
+
+  const rejectResponse = await argsWarden.responseIfRejectingArgs({
+    credsPayload,
+    channelIdentifier,
+    messagePayload,
+  });
+  if (rejectResponse) {
+    return rejectResponse;
+  }
+
+  const creds = await credsFromPayload(credsPayload);
+
+  const {
+    channelName,
+    channelId,
+  } = channelIdentifier;
+
+  const {
+    text,
+    blocks,
+    markdownText,
+  } = messagePayload;
+
+  const response = await slackClient.fetch({
+    url: '/chat.postMessage',
+    method: 'post',
+    body: {
+      channel: channelId || channelName,
+      ...text && { text },
+      ...blocks && { blocks },
+      ...markdownText && { markdown_text: markdownText },
+    },
+    context: { creds },
+    inspect,
+  });
+
+  return response;
+};
+
+const funcApiConfig = {
+  argsWarden,
+};
+
+module.exports = {
+  slackMessagePost,
+  funcApiConfig,
+};
+
+/*
+curl -X POST "http://localhost:8000/slackMessagePost" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "credsPayload": { "credsPath": "slack" },
+    "channelIdentifier": { "channelName": "#hidden_testing" },
+    "messagePayload": { "text": "new number, who dis?" }
+  }'
+*/

package/api/utils.js

@@ -146,14 +146,14 @@ const customFetch = async (url, {
       });
 
       const responseContentType = response.headers.get('content-type');
-      console.log(responseContentType);
+      !HOSTED && console.log('responseContentType', responseContentType);
 
       if (!responseParser) {
         responseParser = getResponseParser(responseContentType);
       }
 
       const parsedResponse = await responseParser(response);
-      logDeep({ parsedResponse });
+      !HOSTED && logDeep({ parsedResponse });
 
       if (response.ok) {
         return {
@@ -278,7 +278,6 @@ const pathAsArray = (path) => {
 
 const objectDigNodeAtPath = (obj, path) => {
   let nodes = pathAsArray(path);
-  console.log(nodes);
 
   let output = obj;
   for (const node of nodes) {
@@ -1161,7 +1160,6 @@ module.exports = {
   actionSingleOrMultiple,
   Processor,
   Getter,
-  capitaliseString,
   sentenceCaseString,
   ArgsWarden,
 };

package/bin/mineral.js

@@ -3,7 +3,7 @@
 const command = process.argv[2];
 
 if (command === 'host') {
-  const { deployFromHostingYml } = require('../_deploy_scripts/deployFromHostingYml');
+  const { deployFromHostingYml } = require('../hosting/deployFromHostingYml');
   deployFromHostingYml().catch((error) => {
     console.error(error);
     process.exit(1);

package/{.hosting.yml.sample → hosting/.hosting.yml.sample}

@@ -4,21 +4,18 @@ google_cloud_info:
 
 # TODO: consider credsPayload in google_cloud_info instead of workspace .creds.yml
 
-env:
-  - HOSTED_API_KEY
-
 functions:
   exampleFunction:
     max_instances: 1
     timeout: 300s
-    wrappers:
-    # entry_point: otherExportName
-      - requireHostedApiKey
+    before_wrappers:
       - allowCrossOriginCallsAndHandleOptions
+      - requireHostedApiKey
+    # entry_point: otherHandlerName
 
   pokemonPokeballThrow:
     max_instances: 1
-    wrappers:
+    before_wrappers:
       - checkTrainer
 
 groups:

package/hosting/copyCredsToEnv.js

@@ -0,0 +1,48 @@
+const fs = require('fs');
+const yaml = require('yaml');
+
+const copyCredsToEnv = (workspace) => {
+  const normalizedWorkspace = workspace.replace(/\/$/, '');
+  const credsPath = `${ normalizedWorkspace }/.creds.yml`;
+  const envPath = `${ normalizedWorkspace }/.env`;
+
+  if (!fs.existsSync(credsPath)) {
+    throw new Error(`Missing .creds.yml in workspace: ${ workspace }`);
+  }
+
+  const credsText = fs.readFileSync(credsPath, 'utf8');
+  const credsFromYml = yaml.parse(credsText);
+  const newCredsLine = `CREDS=${ JSON.stringify(credsFromYml) }`;
+
+  let envFileContents = '';
+  if (fs.existsSync(envPath)) {
+    envFileContents = fs.readFileSync(envPath, 'utf8');
+  }
+
+  if (!envFileContents) {
+    fs.writeFileSync(envPath, `${ newCredsLine }\n`);
+    return;
+  }
+
+  if (/^CREDS=/m.test(envFileContents)) {
+    const updatedFileContents = envFileContents.replace(/^CREDS=.*$/m, newCredsLine);
+
+    if (updatedFileContents !== envFileContents) {
+      fs.writeFileSync(envPath, updatedFileContents);
+    }
+
+    return;
+  }
+
+  fs.appendFileSync(envPath, `\n\n${ newCredsLine }\n`);
+};
+
+if (require.main === module) {
+  const workspace = process.argv[2] || process.cwd();
+  copyCredsToEnv(workspace);
+  console.log('Copied CREDS from .creds.yml to .env');
+}
+
+module.exports = {
+  copyCredsToEnv,
+};

package/{_deploy_scripts → hosting}/deployFromHostingYml.js

@@ -1,14 +1,13 @@
 const readline = require('readline');
-const { toAbsolutePath, setWorkspace } = require('../api/workspace');
+const { toAbsolutePath, setWorkspace, loadWorkspaceEnv } = require('../api/workspace');
 const { getApiDirs, readCliFlag } = require('../cli');
 const { loadHandlers } = require('../server');
 const {
   readHostingYml,
-  getCredsJsonForDeploy,
-  validateEnvVarsForDeploy,
+  ensureWorkspaceEnvForDeploy,
   resolveHostedHandlersForDeploy,
   functionUsesWrapper,
-} = require('../hosting.utils');
+} = require('./hosting.utils');
 const { writeHostedJs } = require('./generateHosted');
 const { execCommand } = require('./execCommand');
 const { formatSetEnvVarsForGcloud, shellQuoteSingle } = require('./setEnvVarsGcloud');
@@ -91,8 +90,6 @@ const deployFunction = async ({
   functionConfig,
   googleCloudInfo,
   workspace,
-  credsJson,
-  deployEnvVars = {},
 }) => {
   const config = {
     ...googleCloudInfo,
@@ -107,20 +104,17 @@ const deployFunction = async ({
     allow_unauthenticated: allowUnauthenticated = true,
     gen2 = true,
     set_env_vars: extraSetEnvVars,
-    entry_point: entryPoint,
+    entry_point,
     schedules,
     groups,
-    wrappers,
+    before_wrappers,
+    after_wrappers,
     source,
+    env,
     ...gcloudArgs
   } = config;
 
-  const resolvedEntryPoint = entryPoint || functionName;
-  const envParts = [
-    'HOSTED=true',
-    `CREDS=${ credsJson }`,
-    ...Object.entries(deployEnvVars).map(([envName, envValue]) => `${ envName }=${ envValue }`),
-  ];
+  const envParts = ['HOSTED=true'];
 
   if (extraSetEnvVars) {
     envParts.push(extraSetEnvVars);
@@ -134,7 +128,7 @@ const deployFunction = async ({
     `--project ${ project }`,
     `--region ${ region }`,
     `--source ${ shellQuoteSingle(workspace) }`,
-    `--entry-point ${ resolvedEntryPoint }`,
+    `--entry-point ${ functionName }`,
     `--trigger-${ trigger }`,
     `--runtime ${ runtime }`,
     `--set-env-vars ${ shellQuoteSingle(setEnvVarsForGcloud) }`,
@@ -157,7 +151,7 @@ const deployFunction = async ({
   }
 
   const requiresHostedApiKey = functionUsesWrapper(functionConfig, 'requireHostedApiKey');
-  const hostedApiKey = deployEnvVars.HOSTED_API_KEY;
+  const hostedApiKey = process.env.HOSTED_API_KEY;
 
   for (const schedule of schedules) {
     const {
@@ -203,17 +197,18 @@ const deployFunction = async ({
 const deployFromHostingYml = async (options = {}) => {
   const config = getHostConfig(options);
   setWorkspace(config.workspace);
+  ensureWorkspaceEnvForDeploy(config.workspace);
+  loadWorkspaceEnv();
 
   const hostingConfig = readHostingYml(config.workspace);
   const {
     google_cloud_info: googleCloudInfo,
-    env: envNames = [],
     functions = {},
     groups = {},
   } = hostingConfig;
 
   if (!googleCloudInfo?.project || !googleCloudInfo?.region) {
-    throw new Error('.hosting.yml requires google_cloud_info.project and google_cloud_info.region');
+    throw new Error('hosting/.hosting.yml requires google_cloud_info.project and google_cloud_info.region');
   }
 
   const handlers = loadHandlers({
@@ -227,19 +222,16 @@ const deployFromHostingYml = async (options = {}) => {
     handlersByName,
   });
 
-  const deployEnvVars = validateEnvVarsForDeploy(config.workspace, envNames);
-
   writeHostedJs({
     workspace: config.workspace,
     hostedHandlers,
   });
 
-  const credsJson = getCredsJsonForDeploy(config.workspace);
   const deployArgs = getDeployArgs();
 
   const deployOne = async (functionName) => {
     if (!functions[functionName]) {
-      console.log(`Function ${ functionName } not found in .hosting.yml, skipping`);
+      console.log(`Function ${ functionName } not found in hosting/.hosting.yml, skipping`);
       return;
     }
 
@@ -248,15 +240,13 @@ const deployFromHostingYml = async (options = {}) => {
       functionConfig: functions[functionName],
       googleCloudInfo,
       workspace: config.workspace,
-      credsJson,
-      deployEnvVars,
     });
   };
 
   if (deployArgs.includes('group')) {
     const groupNames = Object.keys(groups);
     if (!groupNames.length) {
-      console.log('No groups defined in .hosting.yml');
+      console.log('No groups defined in hosting/.hosting.yml');
       return;
     }
 
@@ -270,7 +260,7 @@ const deployFromHostingYml = async (options = {}) => {
   if (deployArgs.includes('function')) {
     const functionNames = Object.keys(functions);
     if (!functionNames.length) {
-      console.log('No functions defined in .hosting.yml');
+      console.log('No functions defined in hosting/.hosting.yml');
       return;
     }
 
@@ -305,14 +295,3 @@ module.exports = {
   deployFromHostingYml,
   getHostConfig,
 };
-
-/*
-  Deploy everything:
-  mineral host all
-
-  Deploy a single function:
-  mineral host function
-
-  Deploy a group of functions:
-  mineral host group
-*/

package/hosting/generateHosted.js

@@ -0,0 +1,87 @@
+const fs = require('fs');
+const { wrapHostedFunction } = require('./hosting.utils');
+
+const formatResolvedWrapperForHostedJs = ({ modulePath, wrapperName }) => (
+  `require('${ modulePath }').${ wrapperName }`
+);
+
+const formatWrapperList = (resolvedWrappers = []) => {
+  if (!resolvedWrappers.length) {
+    return '';
+  }
+
+  const wrapperLines = resolvedWrappers.map((resolvedWrapper) => (
+    `      ${ formatResolvedWrapperForHostedJs(resolvedWrapper) },`
+  ));
+
+  return `\n${ wrapperLines.join('\n') }\n    `;
+};
+
+const formatWrappersArg = ({
+  resolvedBeforeWrappers = [],
+  resolvedAfterWrappers = [],
+} = {}) => {
+  if (!resolvedBeforeWrappers.length && !resolvedAfterWrappers.length) {
+    return '';
+  }
+
+  const beforeWrappersBlock = formatWrapperList(resolvedBeforeWrappers);
+  const afterWrappersBlock = formatWrapperList(resolvedAfterWrappers);
+
+  return `, {
+    beforeWrappers: [${ beforeWrappersBlock }],
+    afterWrappers: [${ afterWrappersBlock }],
+  }`;
+};
+
+const generateHostedJs = ({
+  hostedHandlers,
+}) => {
+  const exportLines = [];
+
+  for (const hostedHandler of hostedHandlers) {
+    const {
+      hostedName,
+      handlerName,
+      resolvedBeforeWrappers = [],
+      resolvedAfterWrappers = [],
+      requirePath,
+    } = hostedHandler;
+    const wrappersArg = formatWrappersArg({
+      resolvedBeforeWrappers,
+      resolvedAfterWrappers,
+    });
+
+    exportLines.push(
+      `  ${ hostedName }: wrapHostedFunction(() => require('${ requirePath }'), '${ handlerName }'${ wrappersArg }),`,
+    );
+  }
+
+  return `// Generated by mineral — do not edit
+const { wrapHostedFunction } = require('@foxtware/mineral/hosting/hosting.utils');
+
+module.exports = {
+${ exportLines.join('\n') }
+};
+`;
+};
+
+const writeHostedJs = ({
+  workspace,
+  hostedHandlers,
+}) => {
+  const { getHostingDir } = require('./hosting.utils');
+  const hostedPath = `${ getHostingDir(workspace) }/hosted.js`;
+  const content = generateHostedJs({
+    hostedHandlers,
+  });
+
+  fs.mkdirSync(getHostingDir(workspace), { recursive: true });
+  fs.writeFileSync(hostedPath, content);
+  return hostedPath;
+};
+
+module.exports = {
+  generateHostedJs,
+  writeHostedJs,
+};

package/{handlerPaths.js → hosting/handlerPaths.js}

@@ -1,4 +1,4 @@
-const MINERAL_ROOT = __dirname;
+const MINERAL_ROOT = `${ __dirname }/..`;
 const MINERAL_API_DIR = `${ MINERAL_ROOT }/api`;
 
 const getRequirePathForHandler = (handler, workspace) => {

package/{hosting.utils.js → hosting/hosting.utils.js}

@@ -1,4 +1,6 @@
 const fs = require('fs');
+const path = require('path');
+const dotenv = require('dotenv');
 const { createRequire } = require('module');
 const yaml = require('yaml');
 const { getRequirePathForHandler } = require('./handlerPaths');
@@ -10,10 +12,16 @@ const {
   funcApi,
   wrapFunction,
   statusCodeFromResult,
-} = require('./server.utils');
+} = require('../server.utils');
 
-const MINERAL_WRAPPERS_MODULE = '@foxtware/mineral/wrappers.js';
-const WORKSPACE_WRAPPERS_MODULE = './wrappers.js';
+dotenv.config({
+  path: path.join(process.cwd(), '.env'),
+});
+
+const MINERAL_WRAPPERS_MODULE = '@foxtware/mineral/hosting/wrappers.js';
+const WORKSPACE_WRAPPERS_MODULE = './hosting/wrappers.js';
+
+const getHostingDir = (workspace) => `${ workspace.replace(/\/$/, '') }/hosting`;
 
 const getMineralWrappers = (workspaceRequire) => {
   try {
@@ -63,13 +71,16 @@ const getHostedEntries = (functions = {}) => (
   Object.entries(functions).map(([hostedName, functionConfig = {}]) => ({
     hostedName,
     handlerName: functionConfig.entry_point || functionConfig.entryPoint || hostedName,
-    wrappers: Array.isArray(functionConfig.wrappers) ? functionConfig.wrappers : [],
+    beforeWrappers: Array.isArray(functionConfig.before_wrappers) ? functionConfig.before_wrappers : [],
+    afterWrappers: Array.isArray(functionConfig.after_wrappers) ? functionConfig.after_wrappers : [],
   }))
 );
 
-const functionUsesWrapper = (functionConfig = {}, wrapperName) => (
-  Array.isArray(functionConfig.wrappers) && functionConfig.wrappers.includes(wrapperName)
-);
+const functionUsesWrapper = (functionConfig = {}, wrapperName) => {
+  const { before_wrappers = [], after_wrappers = [] } = functionConfig;
+
+  return before_wrappers.includes(wrapperName) || after_wrappers.includes(wrapperName);
+};
 
 const getFuncApiConfig = ({
   moduleExports,
@@ -92,7 +103,10 @@ const getFuncApiConfig = ({
   }
 };
 
-const wrapHostedFunction = (loader, exportName, wrappers = []) => {
+const wrapHostedFunction = (loader, exportName, {
+  beforeWrappers = [],
+  afterWrappers = [],
+} = {}) => {
   let handler = null;
   let usesFuncApi = false;
 
@@ -121,7 +135,10 @@ const wrapHostedFunction = (loader, exportName, wrappers = []) => {
       : await handler(...args);
   };
 
-  const wrappedHandler = wrapFunction(coreHandler, wrappers);
+  const wrappedHandler = wrapFunction(coreHandler, {
+    beforeWrappers,
+    afterWrappers,
+  });
 
   return async (req, res) => {
     try {
@@ -156,60 +173,31 @@ const wrapHostedFunction = (loader, exportName, wrappers = []) => {
 };
 
 const readHostingYml = (workspace) => {
-  const hostingPath = `${ workspace }/.hosting.yml`;
+  const hostingPath = `${ getHostingDir(workspace) }/.hosting.yml`;
 
   if (!fs.existsSync(hostingPath)) {
-    throw new Error(`Missing .hosting.yml in workspace: ${ workspace }`);
+    throw new Error(`Missing hosting/.hosting.yml in workspace: ${ workspace }`);
   }
 
   const hostingText = fs.readFileSync(hostingPath, 'utf8');
   const hostingConfig = yaml.parse(hostingText);
 
   if (!hostingConfig || typeof hostingConfig !== 'object' || Array.isArray(hostingConfig)) {
-    throw new Error('Invalid .hosting.yml');
+    throw new Error('Invalid hosting/.hosting.yml');
   }
 
   return hostingConfig;
 };
 
-const getCredsJsonForDeploy = (workspace) => {
-  const credsPath = `${ workspace }/.creds.yml`;
-
-  if (!fs.existsSync(credsPath)) {
-    throw new Error(`Missing .creds.yml in workspace: ${ workspace }`);
-  }
-
-  const credsText = fs.readFileSync(credsPath, 'utf8');
-  return JSON.stringify(yaml.parse(credsText));
-};
+const ensureWorkspaceEnvForDeploy = (workspace) => {
+  const { copyCredsToEnv } = require('./copyCredsToEnv');
+  const envPath = `${ workspace.replace(/\/$/, '') }/.env`;
 
-const getEnvValueForDeploy = (workspace, envName) => {
-  const envPath = `${ workspace }/.env`;
+  copyCredsToEnv(workspace);
 
   if (!fs.existsSync(envPath)) {
-    return '';
+    throw new Error(`Missing .env in workspace: ${ workspace }`);
   }
-
-  const envText = fs.readFileSync(envPath, 'utf8');
-  const match = envText.match(new RegExp(`^${ envName }=(.*)$`, 'm'));
-  return match ? match[1].trim() : '';
-};
-
-const getEnvVarsForDeploy = (workspace, envNames = []) => (
-  Object.fromEntries(
-    envNames.map((envName) => [envName, getEnvValueForDeploy(workspace, envName)]),
-  )
-);
-
-const validateEnvVarsForDeploy = (workspace, envNames = []) => {
-  const envVars = getEnvVarsForDeploy(workspace, envNames);
-  const missing = envNames.filter((envName) => !envVars[envName]);
-
-  if (missing.length) {
-    throw new Error(`Missing required .env values: ${ missing.join(', ') }`);
-  }
-
-  return envVars;
 };
 
 const resolveHostedHandlersForDeploy = ({
@@ -221,9 +209,16 @@ const resolveHostedHandlersForDeploy = ({
   const hostedEntries = getHostedEntries(functions);
 
   return hostedEntries.map((hostedEntry) => {
-    const { handlerName, wrappers = [] } = hostedEntry;
+    const {
+      handlerName,
+      beforeWrappers = [],
+      afterWrappers = [],
+    } = hostedEntry;
 
-    const resolvedWrappers = wrappers.map((wrapperName) => (
+    const resolvedBeforeWrappers = beforeWrappers.map((wrapperName) => (
+      resolveWrapperName(wrapperName, workspaceRequire)
+    ));
+    const resolvedAfterWrappers = afterWrappers.map((wrapperName) => (
       resolveWrapperName(wrapperName, workspaceRequire)
     ));
 
@@ -236,7 +231,8 @@ const resolveHostedHandlersForDeploy = ({
 
     return {
       ...hostedEntry,
-      resolvedWrappers,
+      resolvedBeforeWrappers,
+      resolvedAfterWrappers,
       requirePath: getRequirePathForHandler(handler, workspace),
     };
   });
@@ -245,13 +241,12 @@ const resolveHostedHandlersForDeploy = ({
 // TODO: support credsPayload in google_cloud_info instead of full workspace .creds.yml
 
 module.exports = {
+  getHostingDir,
   resolveWrapperName,
   getFuncApiConfig,
   wrapHostedFunction,
   readHostingYml,
-  getCredsJsonForDeploy,
-  getEnvVarsForDeploy,
-  validateEnvVarsForDeploy,
+  ensureWorkspaceEnvForDeploy,
   getHostedEntries,
   resolveHostedHandlersForDeploy,
   functionUsesWrapper,

package/{wrappers.js → hosting/wrappers.js}

@@ -1,4 +1,4 @@
-const { HOSTED } = require('./api/constants');
+const { HOSTED } = require('../api/constants');
 
 const requireHostedApiKey = async (req) => {
   if (!HOSTED) {
@@ -22,7 +22,7 @@ const allowCrossOriginCallsAndHandleOptions = async (req, res) => {
 
   res.setHeader('Access-Control-Allow-Origin', origin || '*');
   res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
-  res.setHeader('Access-Control-Allow-Headers', 'Content-Type, x-api-key, x-wf-token, x-wf-value');
+  res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, x-api-key, x-wf-token, x-wf-value');
 
   if (req.method === 'OPTIONS') {
     res.writeHead(204);