@foxtware/mineral 0.1.0 → 0.1.2

package/.creds.yml.sample

@@ -8,6 +8,11 @@ peoplevox:
   USERNAME: ______________________________
   PASSWORD: ______________________________
 
+logiwa:
+  BASE_URL: https://myapi.logiwa.com
+  EMAIL: _________________________________
+  PASSWORD: ______________________________
+
 youtube:
   API_KEY: _______________________________
 

package/.gcloudignore

@@ -0,0 +1,17 @@
+# This file specifies files that are *not* uploaded to Google Cloud
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+node_modules
+#!include:.gitignore

package/.hosting.yml.sample

@@ -0,0 +1,23 @@
+google_cloud_info:
+  project:
+  region:
+
+# TODO: consider credsPayload in google_cloud_info instead of workspace .creds.yml
+
+functions:
+  exampleFunction:
+    max_instances: 1
+    timeout: 300s
+    wrappers:
+      - requireHostedApiKey
+    # entry_point: otherExportName
+
+  packagedFunction:
+    source: '@foxtware/mineral/api/pokemon/pokemonPokeballThrow.js'
+    max_instances: 1
+    wrappers:
+      - checkTrainer
+
+groups:
+  example_group:
+    - exampleFunction

package/README.md

@@ -49,4 +49,6 @@ The [bedrock](https://github.com/GorgonFreeman/bedrock) middleware, refactored f
   }
   ```
 - **Monorepo structure**
-Mineral gets pushed to from a larger repo that can also contain private functions. Mineral should be strictly useful stuff for the public, and can be used standalone, but needs to be instantiated for serving, setting stuff like which creds file to use. This allows it to be used as part of another repo in the same HTTP/curl way as by itself. In the serve functions, allow it to operate from a different directory, finding .hosting.yml, .creds.yml, etc. within it.
+Mineral gets pushed to from a larger repo that can also contain private functions. Mineral should be strictly useful stuff for the public, and can be used standalone, but needs to be instantiated for serving, setting stuff like which creds file to use. This allows it to be used as part of another repo in the same HTTP/curl way as by itself. Pass `--workspace` to locate `.creds.yml` and `--api_dirs` to serve additional function directories.
+
+For cloud deploy, workspaces use `.hosting.yml` and `npm run host` (same `--workspace` / `--api_dirs` flags as dev/serve). See `.hosting.yml.sample`.

package/_build_scripts/publish.js

@@ -0,0 +1,105 @@
+const fs = require('fs');
+const path = require('path');
+const readline = require('readline');
+const { execSync } = require('child_process');
+
+const mineralRoot = path.join(__dirname, '..');
+const packageJsonPath = path.join(mineralRoot, 'package.json');
+
+const readPackageJson = () => JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+
+const writePackageJson = (packageJson) => {
+  fs.writeFileSync(packageJsonPath, `${ JSON.stringify(packageJson, null, 2) }\n`);
+};
+
+const parseVersion = (version) => version.split('.').map((part) => Number(part) || 0);
+
+const isVersionGreater = (left, right) => {
+  const [leftMajor, leftMinor, leftPatch] = parseVersion(left);
+  const [rightMajor, rightMinor, rightPatch] = parseVersion(right);
+
+  if (leftMajor !== rightMajor) {
+    return leftMajor > rightMajor;
+  }
+
+  if (leftMinor !== rightMinor) {
+    return leftMinor > rightMinor;
+  }
+
+  return leftPatch > rightPatch;
+};
+
+const bumpPatch = (version) => {
+  const [major, minor, patch] = parseVersion(version);
+  return [major, minor, patch + 1].join('.');
+};
+
+const getPublishedVersion = () => {
+  try {
+    return execSync('npm view @foxtware/mineral version', {
+      cwd: mineralRoot,
+      encoding: 'utf8',
+    }).trim();
+  } catch {
+    return '';
+  }
+};
+
+const ensureLoggedIn = () => {
+  try {
+    execSync('npm whoami', {
+      cwd: mineralRoot,
+      encoding: 'utf8',
+      stdio: 'pipe',
+    });
+  } catch {
+    throw new Error('Not logged in to npm. Run `npm login` with an account that can publish @foxtware/mineral.');
+  }
+};
+
+const askWithDefault = async (question, defaultAnswer) => {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  const answer = await new Promise((resolve) => {
+    rl.question(`${ question } [${ defaultAnswer }] `, resolve);
+  });
+
+  rl.close();
+  return answer.trim() || defaultAnswer;
+};
+
+const publish = async () => {
+  ensureLoggedIn();
+
+  const packageJson = readPackageJson();
+  const localVersion = packageJson.version;
+  const publishedVersion = getPublishedVersion();
+
+  let version = localVersion;
+
+  const localIsAhead = publishedVersion && isVersionGreater(localVersion, publishedVersion);
+
+  if (!localIsAhead) {
+    const latestVersion = publishedVersion || localVersion;
+    const suggestedVersion = bumpPatch(latestVersion);
+    version = await askWithDefault('What version should we use?', suggestedVersion);
+  }
+
+  packageJson.version = version;
+  writePackageJson(packageJson);
+
+  execSync('npm publish', {
+    cwd: mineralRoot,
+    stdio: 'inherit',
+  });
+
+  console.log(`Published @foxtware/mineral@${ version }`);
+};
+
+publish().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});

package/_deploy_scripts/deployFromHostingYml.js

@@ -0,0 +1,319 @@
+const readline = require('readline');
+const { toAbsolutePath, setWorkspace } = require('../api/workspace');
+const { getApiDirs, readCliFlag } = require('../cli');
+const { loadHandlers } = require('../server');
+const { readHostingYml, getCredsJsonForDeploy, getHostedApiKeyForDeploy, resolveHostedHandlersForDeploy, functionUsesWrapper } = require('../hosting.utils');
+const { writeHostedJs } = require('./generateHosted');
+const { execCommand } = require('./execCommand');
+const { formatSetEnvVarsForGcloud, shellQuoteSingle } = require('./setEnvVarsGcloud');
+
+const GCLOUD_NODEJS_MAX_MAJOR = 24;
+
+const gcloudRuntimeFromCurrentNode = () => {
+  const localMajor = Number.parseInt(process.versions.node.split('.')[0], 10);
+  const effective = Number.isNaN(localMajor) ? GCLOUD_NODEJS_MAX_MAJOR : localMajor;
+  const major = Math.min(effective, GCLOUD_NODEJS_MAX_MAJOR);
+  return `nodejs${ major }`;
+};
+
+const getHostConfig = (options = {}) => ({
+  workspace: toAbsolutePath(
+    options.workspace
+      ?? readCliFlag('--workspace')
+      ?? process.env.MINERAL_WORKSPACE
+      ?? process.cwd(),
+  ),
+  api_dirs: getApiDirs(options),
+});
+
+const chooseOption = async (prompt, options) => new Promise((resolve) => {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  console.log(prompt);
+  options.forEach((option, index) => {
+    console.log(`  ${ index + 1 }. ${ option }`);
+  });
+
+  rl.question('> ', (answer) => {
+    rl.close();
+    const choice = Number(answer) - 1;
+    resolve(options[choice] ?? options[0]);
+  });
+});
+
+const anyHostedEntryUsesWrapper = (hostedEntries, wrapperName) => (
+  hostedEntries.some((hostedEntry) => hostedEntry.wrappers?.includes(wrapperName))
+);
+
+const getDeployArgs = () => {
+  const args = process.argv.slice(2);
+  const deployArgs = [];
+
+  for (let index = 0; index < args.length; index++) {
+    const arg = args[index];
+
+    if (arg === '--workspace' || arg === '--api_dirs') {
+      index++;
+      continue;
+    }
+
+    if (arg.startsWith('--workspace=') || arg.startsWith('--api_dirs=')) {
+      continue;
+    }
+
+    deployArgs.push(arg);
+  }
+
+  return deployArgs;
+};
+
+const handlerByRouteName = (handlers) => {
+  const byName = new Map();
+
+  for (const handler of handlers.values()) {
+    byName.set(handler.routeName, handler);
+  }
+
+  return byName;
+};
+
+const deployFunction = async ({
+  functionName,
+  functionConfig,
+  googleCloudInfo,
+  workspace,
+  credsJson,
+  hostedApiKey,
+}) => {
+  const config = {
+    ...googleCloudInfo,
+    ...functionConfig,
+  };
+
+  let {
+    project,
+    region,
+    trigger = 'http',
+    runtime = gcloudRuntimeFromCurrentNode(),
+    allow_unauthenticated: allowUnauthenticated = true,
+    gen2 = true,
+    set_env_vars: extraSetEnvVars,
+    entry_point: entryPoint,
+    schedules,
+    groups,
+    wrappers,
+    source,
+    ...gcloudArgs
+  } = config;
+
+  const resolvedEntryPoint = entryPoint || functionName;
+  const envParts = [
+    'HOSTED=true',
+    `CREDS=${ credsJson }`,
+  ];
+
+  if (hostedApiKey) {
+    envParts.push(`HOSTED_API_KEY=${ hostedApiKey }`);
+  }
+
+  if (extraSetEnvVars) {
+    envParts.push(extraSetEnvVars);
+  }
+
+  const rawSetEnvVars = envParts.join(',');
+  const setEnvVarsForGcloud = formatSetEnvVarsForGcloud(rawSetEnvVars);
+
+  const deployCommand = [
+    `gcloud functions deploy ${ functionName }`,
+    `--project ${ project }`,
+    `--region ${ region }`,
+    `--source ${ shellQuoteSingle(workspace) }`,
+    `--entry-point ${ resolvedEntryPoint }`,
+    `--trigger-${ trigger }`,
+    `--runtime ${ runtime }`,
+    `--set-env-vars ${ shellQuoteSingle(setEnvVarsForGcloud) }`,
+    ...(allowUnauthenticated ? ['--allow-unauthenticated'] : []),
+    ...(gen2 ? ['--gen2'] : []),
+    ...Object.entries(gcloudArgs).map(([key, value]) => `--${ key.replaceAll('_', '-') } ${ value }`),
+  ].join(' ');
+
+  console.log(deployCommand);
+
+  try {
+    await execCommand(deployCommand);
+  } catch (error) {
+    console.error(`Error deploying function ${ functionName }:`, error);
+    return;
+  }
+
+  if (!schedules?.length) {
+    return;
+  }
+
+  const requiresHostedApiKey = functionUsesWrapper(functionConfig, 'requireHostedApiKey');
+
+  for (const schedule of schedules) {
+    const {
+      name: jobName,
+      schedule: jobSchedule,
+      http_method: jobHttpMethod = 'POST',
+      headers: jobHeaders = '',
+      message_body: jobMessageBody,
+      ...schedulerArgs
+    } = schedule;
+
+    let schedulerHeaders = jobHeaders;
+    if (requiresHostedApiKey && hostedApiKey) {
+      schedulerHeaders = schedulerHeaders ? `${ schedulerHeaders },` : '';
+      schedulerHeaders += `x-api-key=${ hostedApiKey }`;
+    }
+
+    try {
+      const checkCommand = `gcloud scheduler jobs describe ${ jobName } --project=${ project } --location=${ region } 2>/dev/null || echo "NOT_FOUND"`;
+      const checkResult = await execCommand(checkCommand);
+      const jobExists = !checkResult.stdout.includes('NOT_FOUND');
+
+      const schedulerCommand = [
+        jobExists ? `gcloud scheduler jobs update http ${ jobName }` : `gcloud scheduler jobs create http ${ jobName }`,
+        `--schedule="${ jobSchedule }"`,
+        `--uri="https://${ region }-${ project }.cloudfunctions.net/${ functionName }"`,
+        `--http-method=${ jobHttpMethod }`,
+        `--project=${ project }`,
+        `--location=${ region }`,
+        jobExists ? `--update-headers ${ schedulerHeaders }` : `--headers ${ schedulerHeaders }`,
+        ...(jobMessageBody ? [`--message-body '${ jobMessageBody }'`] : []),
+        ...Object.entries(schedulerArgs).map(([key, value]) => `--${ key.replaceAll('_', '-') } ${ value }`),
+      ].join(' ');
+
+      console.log(schedulerCommand);
+      await execCommand(schedulerCommand);
+    } catch (error) {
+      console.error(`Error handling scheduler job ${ jobName }:`, error);
+    }
+  }
+};
+
+const deployFromHostingYml = async (options = {}) => {
+  const config = getHostConfig(options);
+  setWorkspace(config.workspace);
+
+  const hostingConfig = readHostingYml(config.workspace);
+  const {
+    google_cloud_info: googleCloudInfo,
+    functions = {},
+    groups = {},
+  } = hostingConfig;
+
+  if (!googleCloudInfo?.project || !googleCloudInfo?.region) {
+    throw new Error('.hosting.yml requires google_cloud_info.project and google_cloud_info.region');
+  }
+
+  const handlers = loadHandlers({
+    ...config,
+    host_mode: true,
+  });
+  const handlersByName = handlerByRouteName(handlers);
+  const hostedHandlers = resolveHostedHandlersForDeploy({
+    functions,
+    workspace: config.workspace,
+    handlersByName,
+  });
+
+  if (anyHostedEntryUsesWrapper(hostedHandlers, 'requireHostedApiKey')) {
+    const hostedApiKey = getHostedApiKeyForDeploy(config.workspace);
+    if (!hostedApiKey) {
+      throw new Error('HOSTED_API_KEY is required in workspace .env when using requireHostedApiKey wrapper');
+    }
+  }
+
+  writeHostedJs({
+    workspace: config.workspace,
+    hostedHandlers,
+  });
+
+  const credsJson = getCredsJsonForDeploy(config.workspace);
+  const hostedApiKey = getHostedApiKeyForDeploy(config.workspace);
+  const deployArgs = getDeployArgs();
+
+  const deployOne = async (functionName) => {
+    if (!functions[functionName]) {
+      console.log(`Function ${ functionName } not found in .hosting.yml, skipping`);
+      return;
+    }
+
+    await deployFunction({
+      functionName,
+      functionConfig: functions[functionName],
+      googleCloudInfo,
+      workspace: config.workspace,
+      credsJson,
+      hostedApiKey,
+    });
+  };
+
+  if (deployArgs.includes('group')) {
+    const groupNames = Object.keys(groups);
+    if (!groupNames.length) {
+      console.log('No groups defined in .hosting.yml');
+      return;
+    }
+
+    const selectedGroup = await chooseOption('Which group would you like to deploy?', groupNames);
+    for (const functionName of groups[selectedGroup] || []) {
+      await deployOne(functionName);
+    }
+    return;
+  }
+
+  if (deployArgs.includes('function')) {
+    const functionNames = Object.keys(functions);
+    if (!functionNames.length) {
+      console.log('No functions defined in .hosting.yml');
+      return;
+    }
+
+    const selectedFunction = await chooseOption('Which function would you like to deploy?', functionNames);
+    await deployOne(selectedFunction);
+    return;
+  }
+
+  if (deployArgs.includes('all')) {
+    for (const functionName of Object.keys(functions)) {
+      await deployOne(functionName);
+    }
+    return;
+  }
+
+  console.log(`
+Usage (from workspace repo):
+  npm run host --workspace . --api_dirs api all
+  npm run host --workspace . --api_dirs api function
+  npm run host --workspace . --api_dirs api group
+`);
+};
+
+if (require.main === module) {
+  deployFromHostingYml().catch((error) => {
+    console.error(error);
+    process.exit(1);
+  });
+}
+
+module.exports = {
+  deployFromHostingYml,
+  getHostConfig,
+};
+
+/*
+  Deploy everything:
+  npm run host all
+
+  Deploy a single function:
+  npm run host function
+
+  Deploy a group of functions:
+  npm run host group
+*/

package/_deploy_scripts/execCommand.js

@@ -0,0 +1,38 @@
+const { spawn } = require('child_process');
+
+const execCommand = (command) => new Promise((resolve, reject) => {
+  const childProcess = spawn(command, [], {
+    stdio: 'pipe',
+    shell: true,
+  });
+
+  let stdout = '';
+  let stderr = '';
+
+  childProcess.stdout.on('data', (data) => {
+    stdout += data.toString();
+    process.stdout.write(data);
+  });
+
+  childProcess.stderr.on('data', (data) => {
+    stderr += data.toString();
+    process.stderr.write(data);
+  });
+
+  childProcess.on('close', (code) => {
+    if (code === 0) {
+      resolve({ stdout, stderr, command });
+      return;
+    }
+
+    reject({ stdout, stderr, command, code });
+  });
+
+  childProcess.on('error', (error) => {
+    reject({ error, command });
+  });
+});
+
+module.exports = {
+  execCommand,
+};

package/_deploy_scripts/generateHosted.js

@@ -0,0 +1,44 @@
+const fs = require('fs');
+const { wrapHostedFunction } = require('../hosting.utils');
+
+const generateHostedJs = ({
+  hostedHandlers,
+}) => {
+  const exportLines = [];
+  const hostingUtilsRequire = '@foxtware/mineral/hosting.utils';
+
+  for (const hostedHandler of hostedHandlers) {
+    const { entryPoint, wrappers = [], requirePath } = hostedHandler;
+    const wrappersArg = wrappers.length ? `, ${ JSON.stringify(wrappers) }` : '';
+
+    exportLines.push(
+      `  ${ entryPoint }: wrapHostedFunction(() => require('${ requirePath }'), '${ entryPoint }'${ wrappersArg }),`,
+    );
+  }
+
+  return `// Generated by mineral — do not edit
+const { wrapHostedFunction } = require('${ hostingUtilsRequire }');
+
+module.exports = {
+${ exportLines.join('\n') }
+};
+`;
+};
+
+const writeHostedJs = ({
+  workspace,
+  hostedHandlers,
+}) => {
+  const hostedPath = `${ workspace.replace(/\/$/, '') }/hosted.js`;
+  const content = generateHostedJs({
+    hostedHandlers,
+  });
+
+  fs.writeFileSync(hostedPath, content);
+  return hostedPath;
+};
+
+module.exports = {
+  generateHostedJs,
+  writeHostedJs,
+};

package/_deploy_scripts/setEnvVarsGcloud.js

@@ -0,0 +1,51 @@
+function splitSetEnvVarsPairs(combined) {
+  const segments = combined.split(/,(?=[A-Za-z_][A-Za-z0-9_]*=)/);
+  return segments.map((segment) => {
+    const eq = segment.indexOf('=');
+    if (eq === -1) {
+      throw new Error(`Invalid set_env_vars segment (no =): ${ segment }`);
+    }
+
+    return {
+      key: segment.slice(0, eq).trim(),
+      value: segment.slice(eq + 1),
+    };
+  });
+}
+
+function pickDelimiter(pairs) {
+  const blob = pairs.map((pair) => `${ pair.key }=${ pair.value }`).join('');
+
+  for (let count = 3; count < 64; count++) {
+    const delimiter = '#'.repeat(count);
+    if (!blob.includes(delimiter)) {
+      return delimiter;
+    }
+  }
+
+  throw new Error('Could not find a delimiter for gcloud --set-env-vars');
+}
+
+function formatSetEnvVarsForGcloud(combined) {
+  const pairs = splitSetEnvVarsPairs(combined);
+  if (pairs.length === 0) {
+    return combined;
+  }
+
+  const needsDelimiter = pairs.length > 1 || pairs.some((pair) => pair.value.includes(','));
+  if (!needsDelimiter) {
+    return combined;
+  }
+
+  const delimiter = pickDelimiter(pairs);
+  return `^${ delimiter }^${ pairs.map((pair) => `${ pair.key }=${ pair.value }`).join(delimiter) }`;
+}
+
+function shellQuoteSingle(value) {
+  return `'${ String(value).replace(/'/g, `'\\''`) }'`;
+}
+
+module.exports = {
+  formatSetEnvVarsForGcloud,
+  shellQuoteSingle,
+};

package/api/logiwa/logiwa.constants.js

@@ -0,0 +1,8 @@
+const DEFAULT_API_VERSION = 'v3.1';
+
+const MAX_PER_PAGE = 200;
+
+module.exports = {
+  DEFAULT_API_VERSION,
+  MAX_PER_PAGE,
+};

package/api/logiwa/logiwa.utils.js

@@ -0,0 +1,53 @@
+const { DEFAULT_API_VERSION } = require('../logiwa/logiwa.constants');
+const { logiwaAuthGet } = require('../logiwa/logiwaAuthGet');
+const {
+  FetchClient,
+  Chain,
+  appendUrlToBase,
+  credsFromPayload,
+  fetchClientCommonSteps,
+} = require('../utils');
+
+const addUrlAndAuthHeaders = async (state) => {
+  const { requestPayload, context } = state;
+  const { credsPayload, apiVersion = DEFAULT_API_VERSION } = context;
+
+  const creds = context.creds ?? await credsFromPayload(credsPayload);
+  const authResponse = await logiwaAuthGet(credsPayload);
+  const { data } = authResponse;
+  const { token } = data;
+
+  // TODO: Handle failed auth
+
+  const { BASE_URL } = creds;
+
+  return {
+    context: {
+      creds,
+    },
+    requestPayload: {
+      url: appendUrlToBase(`${ BASE_URL }/${ apiVersion }`, requestPayload.url),
+      headers: {
+        Authorization: `Bearer ${ token }`,
+        ...requestPayload.headers,
+      },
+    },
+  };
+};
+
+const logiwaClientRequestPreparer = new Chain([
+  addUrlAndAuthHeaders,
+]);
+
+const logiwaClientResponseInterpreter = new Chain([
+  fetchClientCommonSteps.exitEarlyOnNotOk,
+]);
+
+const logiwaClient = new FetchClient({
+  requestPreparer: logiwaClientRequestPreparer,
+  responseInterpreter: logiwaClientResponseInterpreter,
+});
+
+module.exports = {
+  logiwaClient,
+};