@openapi-typescript-infra/service 1.0.0

package/package.json

@@ -0,0 +1,111 @@
+{
+  "name": "@openapi-typescript-infra/service",
+  "version": "1.0.0",
+  "description": "An opinionated framework for building configuration driven services - web, api, or job. Uses OpenAPI, pino logging, express, confit, Typescript and Jest.",
+  "main": "build/index.js",
+  "scripts": {
+    "test": "jest",
+    "lint": "eslint .",
+    "build": "tsc -p tsconfig.build.json && yarn dlx chmodx build/bin/*",
+    "watch": "tsc -p tsconfig.json -w --preserveWatchOutput",
+    "clean": "npx rimraf ./build",
+    "prepublishOnly": "yarn build",
+    "prepack": "pinst --disable",
+    "postpack": "pinst --enable",
+    "_postinstall": "husky install && coconfig"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/openapi-typescript-infra/service.git"
+  },
+  "bin": {
+    "start-service": "./build/bin/start-service.js"
+  },
+  "config": {
+    "coconfig": "@openapi-typescript-infra/coconfig"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "lint-staged": {
+    "*.{js,jsx,ts,tsx}": "yarn eslint --cache --fix"
+  },
+  "keywords": [
+    "service",
+    "openapi",
+    "express",
+    "confit",
+    "babel",
+    "typescript",
+    "jest"
+  ],
+  "author": "developers@pyralis.com>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/openapi-typescript-infra/service/issues"
+  },
+  "homepage": "https://github.com/openapi-typescript-infra/service#readme",
+  "dependencies": {
+    "@gasbuddy/confit": "^3.0.0",
+    "@godaddy/terminus": "^4.12.0",
+    "@opentelemetry/api": "^1.4.1",
+    "@opentelemetry/api-metrics": "^0.33.0",
+    "@opentelemetry/exporter-prometheus": "^0.39.1",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.39.1",
+    "@opentelemetry/instrumentation": "^0.39.1",
+    "@opentelemetry/instrumentation-aws-sdk": "^0.34.2",
+    "@opentelemetry/instrumentation-dns": "^0.31.4",
+    "@opentelemetry/instrumentation-express": "^0.32.3",
+    "@opentelemetry/instrumentation-generic-pool": "^0.31.3",
+    "@opentelemetry/instrumentation-graphql": "^0.34.2",
+    "@opentelemetry/instrumentation-http": "^0.39.1",
+    "@opentelemetry/instrumentation-ioredis": "^0.34.2",
+    "@opentelemetry/instrumentation-net": "^0.31.3",
+    "@opentelemetry/instrumentation-pg": "^0.35.2",
+    "@opentelemetry/instrumentation-pino": "^0.33.3",
+    "@opentelemetry/sdk-metrics": "^1.13.0",
+    "@opentelemetry/sdk-node": "^0.39.1",
+    "@opentelemetry/semantic-conventions": "^1.13.0",
+    "cookie-parser": "^1.4.6",
+    "dotenv": "^16.0.3",
+    "eventsource": "^1.1.2",
+    "express": "next",
+    "express-openapi-validator": "^5.0.4",
+    "glob": "^8.1.0",
+    "lodash": "^4.17.21",
+    "minimist": "^1.2.8",
+    "pino": "^8.14.1",
+    "read-pkg-up": "^7.0.1",
+    "rest-api-support": "^1.16.3",
+    "shortstop-dns": "^1.1.0",
+    "shortstop-handlers": "^1.1.1",
+    "shortstop-yaml": "^1.0.0"
+  },
+  "devDependencies": {
+    "@openapi-typescript-infra/coconfig": "^1.0.0",
+    "@types/cookie-parser": "^1.4.3",
+    "@types/eventsource": "1.1.11",
+    "@types/express": "^4.17.17",
+    "@types/glob": "^8.1.0",
+    "@types/jest": "^29.5.1",
+    "@types/lodash": "^4.14.194",
+    "@types/minimist": "^1.2.2",
+    "@types/node": "^18.16.14",
+    "@types/supertest": "^2.0.12",
+    "coconfig": "^0.10.1",
+    "eslint": "^8.41.0",
+    "eslint-config-gasbuddy": "^7.2.0",
+    "eslint-config-prettier": "^8.8.0",
+    "eslint-plugin-jest": "^27.2.1",
+    "husky": "^8.0.3",
+    "jest": "^29.5.0",
+    "lint-staged": "^13.2.2",
+    "pino-pretty": "^10.0.0",
+    "pinst": "^3.0.0",
+    "supertest": "^6.3.3",
+    "ts-jest": "^29.1.0",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4"
+  },
+  "packageManager": "yarn@3.2.3"
+}

package/src/bin/start-service.ts

@@ -0,0 +1,28 @@
+#!/usr/bin/env node
+import minimist from 'minimist';
+
+import { serviceRepl } from '../development/repl';
+import { isDev } from '../env';
+import { bootstrap } from '../bootstrap';
+
+/**
+ * built - forces the use of the build directory. Defaults to true in stage/prod, not in dev
+ * repl - launch the REPL (defaults to disabling telemetry)
+ * telemetry - whether to use OpenTelemetry. Defaults to false in dev or with repl
+ * nobind - do not listen on http port or expose metrics
+ */
+const argv = minimist(process.argv.slice(2), {
+  boolean: ['built', 'repl', 'telemetry', 'nobind'],
+});
+
+const noTelemetry = (argv.repl || isDev()) && !argv.telemetry;
+bootstrap({
+  ...argv,
+  telemetry: !noTelemetry,
+}).then(({ app, server }) => {
+  if (argv.repl) {
+    serviceRepl(app, () => {
+      server?.close();
+    });
+  }
+});

package/src/bootstrap.ts

@@ -0,0 +1,112 @@
+import path from 'path';
+
+import dotenv from 'dotenv';
+import readPackageUp from 'read-pkg-up';
+import type { NormalizedPackageJson } from 'read-pkg-up';
+
+import type { RequestLocals, ServiceLocals, ServiceStartOptions } from './types';
+import { isDev } from './env';
+import { startWithTelemetry } from './telemetry/index';
+
+interface BootstrapArguments {
+  // The name of the service, else discovered via read-pkg-up
+  name?: string;
+  // The name of the file with the service function, relative to root
+  main?: string;
+  // Root directory of the app, else discovered via read-pkg-up
+  root?: string;
+  // Use built directory. Omitting lets us determine a sensible default
+  built?: boolean;
+  // The location of the package.json used for discovery (defaults to cwd)
+  packageDir?: string;
+  // Whether to engage telemetry
+  telemetry?: boolean;
+  // Don't bind to http port or expose metrics
+  nobind?: boolean;
+}
+
+function resolveMain(packageJson: NormalizedPackageJson) {
+  if (typeof packageJson.main === 'string') {
+    return packageJson.main;
+  }
+  return undefined;
+}
+
+async function getServiceDetails(argv: BootstrapArguments = {}) {
+  if (argv.name && argv.root) {
+    return {
+      rootDirectory: argv.root,
+      name: argv.name,
+      main: argv.main || (isDev() && !argv.built ? 'src/index.ts' : 'build/index.js'),
+    };
+  }
+  const cwd = argv.packageDir ? path.resolve(argv.packageDir) : process.cwd();
+  const pkg = await readPackageUp({ cwd });
+  if (!pkg) {
+    throw new Error(
+      `Unable to find package.json in ${cwd} to get main module. Make sure you are running from the package root directory.`,
+    );
+  }
+  const main = resolveMain(pkg.packageJson);
+  const parts = pkg.packageJson.name.split('/');
+  return {
+    main,
+    rootDirectory: path.dirname(pkg.path),
+    name: parts[parts.length - 1],
+  };
+}
+
+// Automagically start your app by using common patterns
+// to find your implementation and settings. This is most useful
+// for jobs or other scripts that need service infra but are
+// not simply the service
+export async function bootstrap<
+  SLocals extends ServiceLocals = ServiceLocals,
+  RLocals extends RequestLocals = RequestLocals,
+>(argv?: BootstrapArguments) {
+  const { main, rootDirectory, name } = await getServiceDetails(argv);
+
+  let entrypoint: string;
+  let codepath: 'build' | 'src' = 'build';
+  if (isDev() && argv?.built !== true) {
+    // eslint-disable-next-line import/no-extraneous-dependencies
+    const { register } = await import('ts-node');
+    register();
+    if (main) {
+      entrypoint = main.replace(/^(\.?\/?)build\//, '$1src/').replace(/\.js$/, '.ts');
+    } else {
+      entrypoint = './src/index.ts';
+    }
+    codepath = 'src';
+  } else if (main) {
+    entrypoint = main;
+  } else {
+    entrypoint = './build/index.js';
+  }
+
+  dotenv.config();
+
+  const absoluteEntrypoint = path.resolve(rootDirectory, entrypoint);
+  if (argv?.telemetry) {
+    return startWithTelemetry<SLocals, RLocals>({
+      name,
+      rootDirectory,
+      service: absoluteEntrypoint,
+    });
+  }
+
+  // This needs to be required for TS on-the-fly to work
+  // eslint-disable-next-line global-require, import/no-dynamic-require, @typescript-eslint/no-var-requires
+  const impl = require(absoluteEntrypoint);
+  const opts: ServiceStartOptions<SLocals, RLocals> = {
+    name,
+    rootDirectory,
+    service: impl.default || impl.service,
+    codepath,
+  };
+  // eslint-disable-next-line import/no-unresolved
+  const { startApp, listen } = await import('./express-app/app.js');
+  const app = await startApp<SLocals, RLocals>(opts);
+  const server = argv?.nobind ? undefined : await listen(app);
+  return { server, app };
+}

package/src/config/index.ts

@@ -0,0 +1,115 @@
+import fs from 'fs';
+import path from 'path';
+
+import confit from '@gasbuddy/confit';
+
+import { shortstops } from './shortstops';
+import type { ConfigStore } from './types';
+
+// Order matters here.
+const ENVIRONMENTS = ['production', 'staging', 'test', 'development'];
+
+async function pathExists(f: string) {
+  return new Promise((accept, reject) => {
+    fs.stat(f, (err) => {
+      if (!err) {
+        accept(true);
+      } else if (err.code === 'ENOENT') {
+        accept(false);
+      } else {
+        reject(err);
+      }
+    });
+  });
+}
+
+async function addDefaultConfiguration(
+  configFactory: ReturnType<typeof confit>,
+  directory: string,
+  envConfit: ConfigStore,
+) {
+  const addIfEnv = async (e: string) => {
+    const c = path.join(directory, `${e}.json`);
+    if (envConfit.get(`env:${e}`) && (await pathExists(c))) {
+      configFactory.addDefault(c);
+      return true;
+    }
+    return false;
+  };
+
+  await ENVIRONMENTS.reduce(
+    (runningPromise, environment) => runningPromise.then((prev) => prev || addIfEnv(environment)),
+    Promise.resolve(false),
+  );
+
+  const baseConfig = path.join(directory, 'config.json');
+  if (await pathExists(baseConfig)) {
+    configFactory.addDefault(baseConfig);
+  }
+}
+
+export interface ServiceConfigurationSpec {
+  // Used for "sourcerequire" and other source-relative paths and for the package name
+  rootDirectory: string;
+  // The LAST configuration is the most "specific" - if a configuration value
+  // exists in all directories, the last one wins
+  configurationDirectories: string[];
+  name: string;
+}
+
+export async function loadConfiguration({
+  name,
+  configurationDirectories: dirs,
+  rootDirectory,
+}: ServiceConfigurationSpec): Promise<ConfigStore> {
+  const defaultProtocols = shortstops({ name }, rootDirectory);
+  const specificConfig = dirs[dirs.length - 1];
+
+  // This confit version just gets us environment info
+  const envConfit: ConfigStore = await new Promise((accept, reject) => {
+    confit(specificConfig).create((err, config) => (err ? reject(err) : accept(config)));
+  });
+  const configFactory = confit({
+    basedir: specificConfig,
+    protocols: defaultProtocols,
+  });
+
+  /**
+   * Note that in confit, when using addDefault,
+   * the FIRST addDefault takes precendence over the next (and so on), so
+   * if you override this method, you should register your defaults first.
+   */
+  const defaultOrder = dirs.slice(0, dirs.length - 1).reverse();
+  defaultOrder.push(path.join(__dirname, '../..', 'config'));
+  await defaultOrder.reduce(
+    (promise, dir) => promise.then(() => addDefaultConfiguration(configFactory, dir, envConfit)),
+    Promise.resolve(),
+  );
+
+  const loaded: ConfigStore = await new Promise((accept, reject) => {
+    configFactory.create((err, config) => (err ? reject(err) : accept(config)));
+  });
+
+  // TODO init other stuff based on config here, such as key management or
+  // other cloud-aware shortstop handlers
+
+  return loaded;
+}
+
+export function insertConfigurationBefore(
+  configDirs: string[] | undefined,
+  insert: string,
+  before: string,
+) {
+  const copy = [...(configDirs || [])];
+  const index = copy.indexOf(before);
+  if (index === -1) {
+    copy.push(insert, before);
+  } else {
+    copy.splice(index, 0, insert);
+  }
+  return copy;
+}
+
+export * from './schema';
+export * from './types';

package/src/config/schema.ts

@@ -0,0 +1,66 @@
+import type { Level } from 'pino';
+
+export interface ServiceConfiguration {
+  protocol?: string;
+  port?: number;
+  host?: string;
+  basePath?: string;
+  proxy?: string | false;
+}
+
+export interface ConfigurationItemEnabled {
+  enabled?: boolean;
+}
+
+export interface ConfigurationSchema extends Record<string, unknown> {
+  trustProxy?: string[];
+  logging?: {
+    level?: Level;
+    logHttpRequests?: boolean;
+    logRequestBody?: boolean;
+    logResponseBody?: boolean;
+  },
+  routing?: {
+    openapi?: boolean;
+    // Relative to the *root directory* of the app
+    routes?: string;
+    // Whether to add middleware that "freezes" the query string
+    // rather than preserving the new Express@5 behavior of reparsing
+    // every time (which causes problems for OpenAPI validation)
+    freezeQuery?: boolean;
+    // Whether to compute etag headers. http://expressjs.com/en/api.html#etag.options.table
+    etag?: boolean;
+    cookieParser?: boolean;
+    bodyParsers?: {
+      json?: boolean;
+      form?: boolean;
+    },
+    // Set static.enabled to true to enable static assets to be served
+    static?: ConfigurationItemEnabled & {
+      // The path relative to the root directory of the app
+      path?: string;
+      // The path on which to mount the static assets (defaults to /)
+      mountPath?: string;
+    },
+    finalHandlers: {
+      // Whether to create and return errors for unhandled routes
+      notFound?: boolean;
+      // Whether to handle errors and return them to clients
+      // (currently means we will return JSON errors)
+      errors?: ConfigurationItemEnabled & {
+        render?: boolean;
+        // Check to see if we got an error from an upstream
+        // service that has code/domain/message, and if so return
+        // that as is. Otherwise we will sanitize it to avoid leaking
+        // information.
+        unnest: boolean;
+      };
+    },
+  },
+  server?: {
+    internalPort?: number,
+    port?: number,
+    metrics: ConfigurationItemEnabled,
+  },
+  connections: Record<string, ServiceConfiguration>;
+}

package/src/config/shortstops.ts

@@ -0,0 +1,118 @@
+import os from 'os';
+import path from 'path';
+
+import shortstop from 'shortstop-handlers';
+import shortstopYaml from 'shortstop-yaml';
+import shortstopDns from 'shortstop-dns';
+import { ProtocolFn } from '@gasbuddy/confit';
+
+/**
+ * Default shortstop handlers for GasBuddy service configuration
+ */
+
+/**
+ * A require: shortstop that will dig and find a named function
+ * with a url-like hash pattern
+ */
+function betterRequire(basepath: string) {
+  const baseRequire = shortstop.require(basepath);
+  return function hashRequire(v: string) {
+    const [moduleName, func] = v.split('#');
+    const module = baseRequire(moduleName);
+    if (func) {
+      if (module[func]) {
+        return module[func];
+      }
+      return baseRequire(v);
+    }
+    return module;
+  };
+}
+
+/**
+ * Our convention is that service names end with:
+ *  -serv - a back end service not callable by the outside world and where no authorization occurs
+ *  -api - a non-UI front end service that exposes swagger and sometimes non-swagger APIs
+ *  -web - a UI front end service
+ *  -worker - a scheduled job or queue processor
+ *
+ * This shortstop will take a CSV of service types and tell you if this service is
+ * of that type, or if the first character after serviceType: is an exclamation point,
+ * whether it's NOT of any of the specified types
+ */
+function serviceTypeFactory(name: string) {
+  const type = name.split('-').pop();
+
+  return function serviceType(v: string) {
+    let checkValue = v;
+    let matchIsGood = true;
+    if (checkValue[0] === '!') {
+      matchIsGood = false;
+      checkValue = checkValue.substring(1);
+    }
+    const values = checkValue.split(',');
+    // Welp, there's no XOR so here we are.
+    return type && values.includes(type) ? matchIsGood : !matchIsGood;
+  };
+}
+
+const osMethods = {
+  hostname: os.hostname,
+  platform: os.platform,
+  type: os.type,
+  version: os.version,
+};
+
+export function shortstops(
+  service: { name: string; },
+  sourcedir: string,
+) {
+  /**
+   * Since we use transpiled sources a lot,
+   * basedir and sourcedir are meaningfully different reference points.
+   */
+  const basedir = path.join(sourcedir, '..');
+
+  const env = shortstop.env();
+
+  return {
+    env,
+    // A version of env that can default to false
+    env_switch(v: string) {
+      if (v && v[0] === '!') {
+        const bval = env(`${v.substring(1)}|b`);
+        return !bval;
+      }
+      return !!env(v);
+    },
+    base64: shortstop.base64(),
+    regex(v: string) {
+      const [, pattern, flags] = v.match(/^\/(.*)\/([a-z]*)/) || [];
+      return new RegExp(pattern, flags);
+    },
+
+    // handle source and base directory intelligently
+    path: shortstop.path(basedir),
+    sourcepath: shortstop.path(sourcedir),
+    file: shortstop.file(basedir),
+    sourcefile: shortstop.file(sourcedir),
+    require: betterRequire(basedir),
+    sourcerequire: betterRequire(sourcedir),
+
+    // Sometimes yaml is more pleasant for configuration
+    yaml: shortstopYaml(basedir) as ProtocolFn<ReturnType<typeof JSON.parse>>,
+
+    // Switch on service type
+    servicetype: serviceTypeFactory(service.name),
+    servicename: (v: string) => v.replace(/\$\{name\}/g, service.name),
+
+    os(p: keyof typeof osMethods) {
+      return osMethods[p]();
+    },
+    dns: shortstopDns() as ProtocolFn<string>,
+    // No-op in case you have values that start with a shortstop handler name (and colon)
+    literal(v: string) {
+      return v;
+    },
+  };
+}

package/src/config/types.ts

@@ -0,0 +1,5 @@
+export interface ConfigStore {
+  // Confit supports more things (set, use), but that's not how we
+  // intend it to be used.
+  get<T>(name: string): T | undefined;
+}

package/src/development/port-finder.ts

@@ -0,0 +1,40 @@
+import net from 'net';
+
+// Inspired by https://github.com/kessler/find-port/blob/master/lib/findPort.js
+async function isAvailable(port: number) {
+  return new Promise((accept, reject) => {
+    const server = net.createServer().listen(port);
+
+    const timeoutRef = setTimeout(() => {
+      accept(false);
+    }, 2000);
+
+    timeoutRef.unref();
+
+    server.once('listening', () => {
+      clearTimeout(timeoutRef);
+      server.close();
+      accept(true);
+    });
+    server.once('error', (err) => {
+      clearTimeout(timeoutRef);
+
+      if ((err as { code?: string; }).code === 'EADDRINUSE') {
+        accept(false);
+        return;
+      }
+
+      reject(err);
+    });
+  });
+}
+
+export async function findPort(start: number) {
+  for (let p = start; p < start + 1000; p += 1) {
+    // eslint-disable-next-line no-await-in-loop
+    if (await isAvailable(p)) {
+      return p;
+    }
+  }
+  return null;
+}

package/src/development/repl.ts

@@ -0,0 +1,24 @@
+import repl from 'repl';
+import path from 'path';
+
+import { ServiceExpress } from '../types';
+
+export function serviceRepl(app: ServiceExpress, onExit: () => void) {
+  const rl = repl.start({
+    prompt: '> ',
+  });
+  Object.assign(rl.context, app.locals, {
+    app,
+    dump(o: unknown) {
+      // eslint-disable-next-line no-console
+      console.log(JSON.stringify(o, null, '\t'));
+    },
+  });
+  rl.setupHistory(path.resolve('.node_repl_history'), (err) => {
+    if (err) {
+      // eslint-disable-next-line no-console
+      console.error('History setup failed', err);
+    }
+  });
+  rl.on('exit', onExit);
+}

package/src/env.ts

@@ -0,0 +1,14 @@
+export function getNodeEnv() {
+  switch (process.env.APP_ENV || process.env.NODE_ENV) {
+    case 'production':
+    case 'staging':
+    case 'test':
+      return process.env.APP_ENV || process.env.NODE_ENV;
+    default:
+      return 'development';
+  }
+}
+
+export function isDev() {
+  return getNodeEnv() === 'development';
+}

package/src/error.ts

@@ -0,0 +1,44 @@
+import type { ServiceLike, ServiceLocals } from './types';
+
+export interface ServiceErrorSpec {
+  status?: number;
+  code?: string;
+  domain?: string;
+  display_message?: string;
+  log_stack?: boolean;
+  expected_error?: boolean;
+}
+
+/**
+ * An error that gives more structured information to callers. Throw inside a handler as
+ *
+ *   throw new Error(req, 'Something broke', { code: 'SomethingBroke', status: 400 });
+ *
+ * You can also include a display_message which is intended to be viewed by the end user
+ */
+export class ServiceError extends Error {
+  public status: number | undefined;
+
+  public code?: string;
+
+  public domain: string;
+
+  public display_message?: string;
+
+  public log_stack?: boolean;
+
+  // If true, this shouldn't be logged as an error, but as an info log.
+  // This is common when the error needs to go to the client, but should not
+  // take up the valuable mental space of an error log.
+  public expected_error?: boolean;
+
+  constructor(
+    app: ServiceLike<ServiceLocals>,
+    message: string,
+    spec?: ServiceErrorSpec,
+  ) {
+    super(message);
+    this.domain = app.locals.name;
+    Object.assign(this, spec);
+  }
+}