@foxtware/mineral 0.1.7 → 0.1.9

package/hosting/copyCredsToEnv.js

@@ -1,7 +1,59 @@
 const fs = require('fs');
 const yaml = require('yaml');
 
-const copyCredsToEnv = (workspace) => {
+const getValueAtCredsPath = (creds, pathParts) => {
+  let current = creds;
+
+  for (const pathPart of pathParts) {
+    if (current == null || typeof current !== 'object' || !(pathPart in current)) {
+      return undefined;
+    }
+
+    current = current[pathPart];
+  }
+
+  return current;
+};
+
+const setValueAtCredsPath = (target, pathParts, value) => {
+  let current = target;
+
+  for (let index = 0; index < pathParts.length - 1; index++) {
+    const pathPart = pathParts[index];
+
+    if (!(pathPart in current) || typeof current[pathPart] !== 'object' || current[pathPart] === null) {
+      current[pathPart] = {};
+    }
+
+    current = current[pathPart];
+  }
+
+  current[pathParts[pathParts.length - 1]] = value;
+};
+
+const filterCredsByPaths = (creds, includeCredsPaths) => {
+  const filteredCreds = {};
+
+  for (const credsPath of includeCredsPaths) {
+    if (typeof credsPath !== 'string' || !credsPath.trim()) {
+      throw new Error(`Invalid include_creds_paths entry: ${ credsPath }`);
+    }
+
+    const pathParts = credsPath.split('.');
+    const value = getValueAtCredsPath(creds, pathParts);
+
+    if (value === undefined) {
+      throw new Error(`Missing creds path "${ credsPath }" in .creds.yml`);
+    }
+
+    setValueAtCredsPath(filteredCreds, pathParts, value);
+  }
+
+  return filteredCreds;
+};
+
+const copyCredsToEnv = (workspace, options = {}) => {
+  const { includeCredsPaths } = options;
   const normalizedWorkspace = workspace.replace(/\/$/, '');
   const credsPath = `${ normalizedWorkspace }/.creds.yml`;
   const envPath = `${ normalizedWorkspace }/.env`;
@@ -12,7 +64,10 @@ const copyCredsToEnv = (workspace) => {
 
   const credsText = fs.readFileSync(credsPath, 'utf8');
   const credsFromYml = yaml.parse(credsText);
-  const newCredsLine = `CREDS=${ JSON.stringify(credsFromYml) }`;
+  const credsForEnv = includeCredsPaths?.length
+    ? filterCredsByPaths(credsFromYml, includeCredsPaths)
+    : credsFromYml;
+  const newCredsLine = `CREDS=${ JSON.stringify(credsForEnv) }`;
 
   let envFileContents = '';
   if (fs.existsSync(envPath)) {
@@ -45,4 +100,5 @@ if (require.main === module) {
 
 module.exports = {
   copyCredsToEnv,
+  filterCredsByPaths,
 };

package/hosting/deployFromHostingYml.js

@@ -109,6 +109,7 @@ const deployFunction = async ({
     groups,
     before_wrappers,
     after_wrappers,
+    include_creds_paths,
     source,
     env,
     ...gcloudArgs
@@ -197,7 +198,6 @@ const deployFunction = async ({
 const deployFromHostingYml = async (options = {}) => {
   const config = getHostConfig(options);
   setWorkspace(config.workspace);
-  ensureWorkspaceEnvForDeploy(config.workspace);
   loadWorkspaceEnv();
 
   const hostingConfig = readHostingYml(config.workspace);
@@ -235,9 +235,16 @@ const deployFromHostingYml = async (options = {}) => {
       return;
     }
 
+    const functionConfig = functions[functionName];
+    const includeCredsPaths = Array.isArray(functionConfig.include_creds_paths)
+      ? functionConfig.include_creds_paths
+      : undefined;
+
+    ensureWorkspaceEnvForDeploy(config.workspace, { includeCredsPaths });
+
     await deployFunction({
       functionName,
-      functionConfig: functions[functionName],
+      functionConfig,
       googleCloudInfo,
       workspace: config.workspace,
     });

package/hosting/generateHosted.js

@@ -1,8 +1,20 @@
 const fs = require('fs');
 const { wrapHostedFunction } = require('./hosting.utils');
 
+const requirePathForHostedJs = (modulePath) => {
+  if (modulePath === './hosting/wrappers.js') {
+    return './wrappers.js';
+  }
+
+  if (modulePath.startsWith('./api/')) {
+    return `../${ modulePath.slice(2) }`;
+  }
+
+  return modulePath;
+};
+
 const formatResolvedWrapperForHostedJs = ({ modulePath, wrapperName }) => (
-  `require('${ modulePath }').${ wrapperName }`
+  `require('${ requirePathForHostedJs(modulePath) }').${ wrapperName }`
 );
 
 const formatWrapperList = (resolvedWrappers = []) => {
@@ -53,7 +65,7 @@ const generateHostedJs = ({
     });
 
     exportLines.push(
-      `  ${ hostedName }: wrapHostedFunction(() => require('${ requirePath }'), '${ handlerName }'${ wrappersArg }),`,
+      `  ${ hostedName }: wrapHostedFunction(() => require('${ requirePathForHostedJs(requirePath) }'), '${ handlerName }'${ wrappersArg }),`,
     );
   }
 
@@ -77,6 +89,14 @@ const writeHostedJs = ({
   });
 
   fs.mkdirSync(getHostingDir(workspace), { recursive: true });
+
+  if (fs.existsSync(hostedPath)) {
+    const existingContent = fs.readFileSync(hostedPath, 'utf8');
+    if (existingContent === content) {
+      return hostedPath;
+    }
+  }
+
   fs.writeFileSync(hostedPath, content);
   return hostedPath;
 };

package/hosting/hosting.utils.js

@@ -189,11 +189,12 @@ const readHostingYml = (workspace) => {
   return hostingConfig;
 };
 
-const ensureWorkspaceEnvForDeploy = (workspace) => {
+const ensureWorkspaceEnvForDeploy = (workspace, options = {}) => {
+  const { includeCredsPaths } = options;
   const { copyCredsToEnv } = require('./copyCredsToEnv');
   const envPath = `${ workspace.replace(/\/$/, '') }/.env`;
 
-  copyCredsToEnv(workspace);
+  copyCredsToEnv(workspace, { includeCredsPaths });
 
   if (!fs.existsSync(envPath)) {
     throw new Error(`Missing .env in workspace: ${ workspace }`);

package/hosting/hostingPreview.js

@@ -0,0 +1,160 @@
+const http = require('http');
+const { createRequire } = require('module');
+const { respondJson, errorToReadable } = require('../server.utils');
+const { getWorkspace, setWorkspace, loadWorkspaceEnv, toAbsolutePath } = require('../api/workspace');
+const { getApiDirs, readCliFlag } = require('../cli');
+const { loadHandlers } = require('../server');
+const {
+  readHostingYml,
+  resolveHostedHandlersForDeploy,
+  wrapHostedFunction,
+} = require('./hosting.utils');
+const { copyCredsToEnv } = require('./copyCredsToEnv');
+
+const getConfig = (options = {}) => ({
+  port: Number(options.port ?? process.env.PORT ?? 8000),
+  workspace: toAbsolutePath(
+    options.workspace
+      ?? readCliFlag('--workspace')
+      ?? process.env.MINERAL_WORKSPACE
+      ?? process.cwd(),
+  ),
+  api_dirs: getApiDirs(options),
+});
+
+const handlerByRouteName = (handlers) => {
+  const byName = new Map();
+
+  for (const handler of handlers.values()) {
+    byName.set(handler.routeName, handler);
+  }
+
+  return byName;
+};
+
+const loadHostedRoutes = (config) => {
+  const { functions } = readHostingYml(config.workspace);
+  const handlers = loadHandlers({
+    ...config,
+    host_mode: true,
+  });
+  const handlersByName = handlerByRouteName(handlers);
+  const hostedHandlers = resolveHostedHandlersForDeploy({
+    functions,
+    workspace: config.workspace,
+    handlersByName,
+  });
+
+  const workspaceRequire = createRequire(`${ config.workspace.replace(/\/$/, '') }/package.json`);
+  const routes = new Map();
+
+  for (const hostedHandler of hostedHandlers) {
+    const {
+      hostedName,
+      handlerName,
+      resolvedBeforeWrappers = [],
+      resolvedAfterWrappers = [],
+      requirePath,
+    } = hostedHandler;
+
+    const beforeWrappers = resolvedBeforeWrappers.map(({ modulePath, wrapperName }) => (
+      workspaceRequire(modulePath)[wrapperName]
+    ));
+    const afterWrappers = resolvedAfterWrappers.map(({ modulePath, wrapperName }) => (
+      workspaceRequire(modulePath)[wrapperName]
+    ));
+
+    routes.set(
+      `/${ hostedName }`,
+      wrapHostedFunction(
+        () => workspaceRequire(requirePath),
+        handlerName,
+        {
+          beforeWrappers,
+          afterWrappers,
+        },
+      ),
+    );
+  }
+
+  return routes;
+};
+
+const createHostedPreviewServer = (handlers) => http.createServer(async (req, res) => {
+  if (req.method === 'GET' && req.url === '/') {
+    respondJson(res, 200, {
+      ok: true,
+      data: {
+        mode: 'hosting_preview',
+        routes: [...handlers.keys()].sort(),
+        workspace: getWorkspace(),
+      },
+    });
+    return;
+  }
+
+  const route = (req.url || '/').split('?')[0];
+  const handler = handlers.get(route);
+
+  if (!handler) {
+    respondJson(res, 404, {
+      ok: false,
+      error: {
+        code: 'NOT_FOUND',
+        message: `No hosted handler found for route ${ route }`,
+      },
+    });
+    return;
+  }
+
+  try {
+    await handler(req, res);
+  } catch (error) {
+    if (res.headersSent) {
+      return;
+    }
+
+    respondJson(res, 500, {
+      ok: false,
+      error: {
+        code: 'UNHANDLED_ERROR',
+        message: 'Unhandled server error.',
+        details: errorToReadable(error),
+      },
+    });
+  }
+});
+
+const startHostingPreview = (options = {}) => {
+  process.env.HOSTED = 'true';
+
+  const config = getConfig(options);
+
+  setWorkspace(config.workspace);
+  copyCredsToEnv(config.workspace);
+  loadWorkspaceEnv();
+
+  const handlers = loadHostedRoutes(config);
+  const server = createHostedPreviewServer(handlers);
+
+  server.listen(config.port, () => {
+    console.log(`Hosting preview running on port ${ config.port }`);
+    console.log(`Workspace: ${ config.workspace }`);
+    console.log('Hosted routes:');
+
+    for (const route of [...handlers.keys()].sort()) {
+      console.log(route);
+    }
+  });
+
+  return server;
+};
+
+module.exports = {
+  startHostingPreview,
+  loadHostedRoutes,
+};
+
+if (require.main === module) {
+  startHostingPreview();
+}

package/hosting/wrappers.js

@@ -22,7 +22,7 @@ const allowCrossOriginCallsAndHandleOptions = async (req, res) => {
 
   res.setHeader('Access-Control-Allow-Origin', origin || '*');
   res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
-  res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, x-api-key, x-wf-token, x-wf-value');
+  res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, x-api-key, x-wf-token, x-wf-value, x-wf-app');
 
   if (req.method === 'OPTIONS') {
     res.writeHead(204);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@foxtware/mineral",
-  "version": "0.1.7",
+  "version": "0.1.9",
   "bin": {
     "mineral": "bin/mineral.js"
   },
@@ -10,17 +10,24 @@
   },
   "scripts": {
     "creds_to_env": "node hosting/copyCredsToEnv.js",
-    "dev": "node --watch --watch-path=./api --watch-path=./server.js --watch-path=./server.utils.js --watch-path=./.creds.yml server.js",
+    "dev": "node --watch --watch-path=./api --watch-path=./server.js --watch-path=./server.utils.js --watch-path=./hosting --watch-path=./.creds.yml server.js",
+    "hosting_preview": "node --watch --watch-path=./api --watch-path=./hosting/.hosting.yml --watch-path=./hosting/wrappers.js --watch-path=./.creds.yml hosting/hostingPreview.js",
+    "tunnel": "ngrok http http://localhost:8000",
     "new": "node _build_scripts/createNewFunction.js",
     "serve": "PORT=8100 node server.js",
     "start": "node server.js",
     "npm:publish": "node _build_scripts/publish.js"
   },
   "dependencies": {
+    "@supabase/supabase-js": "^2.55.0",
+    "@upstash/redis": "^1.28.4",
     "csvtojson": "^2.0.14",
     "dotenv": "^17.2.0",
     "json-2-csv": "^5.5.9",
     "xml2js": "^0.6.2",
     "yaml": "^2.9.0"
+  },
+  "devDependencies": {
+    "ngrok": "^5.0.0-beta.2"
   }
 }

package/server.utils.js

@@ -258,6 +258,7 @@ const funcApi = (func, config = {}) => {
 
     let callArgs = requestContext.args;
     if (passThroughReq) {
+      requestContext.req.body = modifiedBody;
       callArgs = [requestContext.req];
     } else if (passThroughBody) {
       callArgs = [modifiedBody];