@openapi-typescript-infra/service 1.0.0

package/src/service-calls/index.ts

@@ -0,0 +1,129 @@
+import { URL } from 'node:url';
+
+import type { FetchConfig, FetchRequest, RestApiResponse } from 'rest-api-support';
+import EventSource from 'eventsource';
+
+import { ServiceError, ServiceErrorSpec } from '../error';
+import type {
+  ServiceExpress,
+  ServiceLike,
+  ServiceLocals,
+} from '../types';
+import type { ServiceConfiguration } from '../config/schema';
+
+type UntypedEventSourceHandler = Parameters<EventSource['addEventListener']>[1];
+
+class CustomEventSource extends EventSource {
+  private activeListeners: Array<{ handler: UntypedEventSourceHandler; name: string }> = [];
+
+  addEventListener<T>(name: string, handler: (data: MessageEvent<T>) => void): this {
+    super.addEventListener(name, handler);
+    this.activeListeners.push({ name, handler });
+    return this;
+  }
+
+  removeAllListeners() {
+    this.activeListeners.forEach((l) => {
+      super.removeEventListener(l.name as keyof EventSourceEventMap, l.handler);
+    });
+  }
+}
+
+/**
+ * Return a factory that will make instances of an OpenAPI/Swagger client for each request
+ */
+export function createServiceInterface<ServiceType>(
+  service: ServiceExpress,
+  name: string,
+  Implementation: { new (c: FetchConfig): ServiceType },
+): ServiceType {
+  const appConfig = service.locals.config;
+  const config = {
+    ...(appConfig.get('connections:default') || {}),
+    ...(appConfig.get(`connections:${name}`) || {}),
+  } as ServiceConfiguration;
+  const protocol = config?.protocol || 'http';
+  const port = config?.port || 8000;
+  const host = config?.host || name;
+  const baseUrl = `${protocol}${protocol.endsWith(':') ? '//' : '://'}${host}:${port}${
+    config?.basePath || ''
+  }`;
+
+  const fetchConfig: FetchConfig = {
+    fetch,
+    AbortController,
+    EventSource: CustomEventSource,
+    FormData,
+    baseUrl,
+  };
+
+  // In development, it can be useful to route requests through
+  // a centralized local proxy (we use https://github.com/gas-buddy/container-proxy).
+  // This allows you to run a subset of services locally and route the rest
+  // of the requests to another (typically remote) environment.
+  if (config?.proxy) {
+    const proxyUrl = new URL(config.proxy);
+    const proxyPort = proxyUrl.protocol === 'https:' ? '8443' : '8000';
+
+    fetchConfig.requestInterceptor = (params: FetchRequest) => {
+      const parsedUrl = new URL(params.url);
+      const proto = parsedUrl.protocol.replace(/:$/, '');
+      const defaultPort = proto === 'https' ? 8443 : 8000;
+      const headers: FetchRequest['headers'] = {};
+      headers.host = `${proto}.${parsedUrl.hostname}.${port || defaultPort}`;
+      headers.source = service.locals.name;
+      parsedUrl.hostname = proxyUrl.hostname;
+      parsedUrl.protocol = proxyUrl.protocol;
+      parsedUrl.port = proxyUrl.port || proxyPort;
+      // eslint-disable-next-line no-param-reassign
+      params.headers = params.headers || {};
+      Object.assign(params.headers, headers);
+      // eslint-disable-next-line no-param-reassign
+      params.url = parsedUrl.href;
+    };
+  }
+
+  return new Implementation(fetchConfig);
+}
+
+interface SpecWithMessage extends ServiceErrorSpec {
+  message?: string;
+}
+
+function readResponse<
+  SLocals extends ServiceLocals,
+  AppType extends ServiceLike<SLocals>,
+  ResType extends RestApiResponse<number, SuccessResponseType>,
+  SuccessResponseType,
+>(
+  app: AppType,
+  response: ResType,
+  errorSpec?: SpecWithMessage,
+): Extract<ResType, { responseType: 'response' }> {
+  if (response.responseType === 'response') {
+    return response as Extract<ResType, { responseType: 'response' }>;
+  }
+  const { message, ...spec } = errorSpec || {};
+  throw new ServiceError(
+    app,
+    message || (response.body as Error).message || 'Internal Error',
+    {
+      status: response.status,
+      ...spec,
+    },
+  );
+}
+
+export async function throwOrGetResponse<
+  SLocals extends ServiceLocals,
+  AppType extends ServiceLike<SLocals>,
+  ResType extends RestApiResponse<number, SuccessResponseType>,
+  SuccessResponseType,
+>(
+  app: AppType,
+  exec: () => Promise<ResType>,
+  errorSpec?: SpecWithMessage,
+): Promise<Extract<ResType, { responseType: 'response' }>> {
+  const response = await exec();
+  return readResponse(app, response, errorSpec);
+}

package/src/telemetry/fetchInstrumentation.ts

@@ -0,0 +1,209 @@
+import diagch from 'node:diagnostics_channel';
+
+import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
+import { Instrumentation, InstrumentationConfig } from '@opentelemetry/instrumentation';
+import {
+  Attributes,
+  context,
+  propagation,
+  Span,
+  SpanKind,
+  SpanStatusCode,
+  trace,
+  Tracer,
+  TracerProvider,
+} from '@opentelemetry/api';
+import { Meter, MeterProvider, metrics } from '@opentelemetry/api-metrics';
+
+interface ListenerRecord {
+  name: string;
+  channel: diagch.Channel;
+  onMessage: diagch.ChannelListener;
+}
+
+interface FetchRequestArguments {
+  request: {
+    path: string;
+    method: string;
+    origin: string;
+    headers: string;
+  };
+  span: Span;
+  additionalHeaders: Record<string, unknown>;
+  error?: Error;
+}
+
+type ErrorFetchRequestArguments = FetchRequestArguments & { error: Error; };
+type ResponseFetchRequestArguments = FetchRequestArguments & {
+  response: {
+    statusCode: number;
+    headers: Buffer[];
+  };
+};
+
+interface FetchInstrumentationConfig extends InstrumentationConfig {
+  onRequest?: (args: FetchRequestArguments) => void;
+}
+
+// Get the content-length from undici response headers.
+// `headers` is an Array of buffers: [k, v, k, v, ...].
+// If the header is not present, or has an invalid value, this returns null.
+function contentLengthFromResponseHeaders(headers: Buffer[]) {
+  const name = 'content-length';
+  for (let i = 0; i < headers.length; i += 2) {
+    const k = headers[i];
+    if (k.length === name.length && k.toString().toLowerCase() === name) {
+      const v = Number(headers[i + 1]);
+      if (!Number.isNaN(Number(v))) {
+        return v;
+      }
+      return undefined;
+    }
+  }
+  return undefined;
+}
+
+// A combination of https://github.com/elastic/apm-agent-nodejs and
+// https://github.com/gadget-inc/opentelemetry-instrumentations/blob/main/packages/opentelemetry-instrumentation-undici/src/index.ts
+export class FetchInstrumentation implements Instrumentation {
+  // Keep ref to avoid https://github.com/nodejs/node/issues/42170 bug and for
+  // unsubscribing.
+  private channelSubs: ListenerRecord[];
+
+  private spanFromReq = new WeakMap<object, Span>();
+
+  private tracer: Tracer;
+
+  private config: FetchInstrumentationConfig;
+
+  private meter: Meter;
+
+  public readonly instrumentationName = 'opentelemetry-instrumentation-node-18-fetch';
+
+  public readonly instrumentationVersion = '1.0.0';
+
+  public readonly instrumentationDescription = 'Instrumentation for Node 18 fetch via diagnostics_channel';
+
+  private subscribeToChannel(diagnosticChannel: string, onMessage: diagch.ChannelListener) {
+    const channel = diagch.channel(diagnosticChannel);
+    channel.subscribe(onMessage);
+    this.channelSubs.push({
+      name: diagnosticChannel,
+      channel,
+      onMessage,
+    });
+  }
+
+  constructor(config: FetchInstrumentationConfig) {
+    // Force load fetch API (since it's lazy loaded in Node 18)
+    fetch('').catch(Promise.resolve);
+    this.channelSubs = [];
+    this.meter = metrics.getMeter(this.instrumentationName, this.instrumentationVersion);
+    this.tracer = trace.getTracer(this.instrumentationName, this.instrumentationVersion);
+    this.config = { ...config };
+  }
+
+  disable(): void {
+    this.channelSubs?.forEach((sub) => sub.channel.unsubscribe(sub.onMessage));
+  }
+
+  enable(): void {
+    this.subscribeToChannel('undici:request:create', (args) => this.onRequest(args as FetchRequestArguments));
+    this.subscribeToChannel('undici:request:headers', (args) => this.onHeaders(args as ResponseFetchRequestArguments));
+    this.subscribeToChannel('undici:request:trailers', (args) => this.onDone(args as FetchRequestArguments));
+    this.subscribeToChannel('undici:request:error', (args) => this.onError(args as ErrorFetchRequestArguments));
+  }
+
+  setTracerProvider(tracerProvider: TracerProvider): void {
+    this.tracer = tracerProvider.getTracer(
+      this.instrumentationName,
+      this.instrumentationVersion,
+    );
+  }
+
+  public setMeterProvider(meterProvider: MeterProvider): void {
+    this.meter = meterProvider.getMeter(
+      this.instrumentationName,
+      this.instrumentationVersion,
+    );
+  }
+
+  setConfig(config: InstrumentationConfig): void {
+    this.config = { ...config };
+  }
+
+  getConfig(): InstrumentationConfig {
+    return this.config;
+  }
+
+  onRequest({ request }: FetchRequestArguments): void {
+    // We do not handle instrumenting HTTP CONNECT. See limitation notes above.
+    if (request.method === 'CONNECT') {
+      return;
+    }
+    const span = this.tracer.startSpan(`HTTP ${request.method}`, {
+      kind: SpanKind.CLIENT,
+      attributes: {
+        [SemanticAttributes.HTTP_URL]: String(request.origin),
+        [SemanticAttributes.HTTP_METHOD]: request.method,
+        [SemanticAttributes.HTTP_TARGET]: request.path,
+        'http.client': 'fetch',
+      },
+    });
+    const requestContext = trace.setSpan(context.active(), span);
+    const addedHeaders: Record<string, string> = {};
+    propagation.inject(requestContext, addedHeaders);
+
+    if (this.config.onRequest) {
+      this.config.onRequest({ request, span, additionalHeaders: addedHeaders });
+    }
+
+    request.headers += Object.entries(addedHeaders)
+      .map(([k, v]) => `${k}: ${v}\r\n`)
+      .join('');
+    this.spanFromReq.set(request, span);
+  }
+
+  onHeaders({ request, response }: ResponseFetchRequestArguments): void {
+    const span = this.spanFromReq.get(request);
+
+    if (span !== undefined) {
+      // We are currently *not* capturing response headers, even though the
+      // intake API does allow it, because none of the other `setHttpContext`
+      // uses currently do.
+
+      const cLen = contentLengthFromResponseHeaders(response.headers);
+      const attrs: Attributes = {
+        [SemanticAttributes.HTTP_STATUS_CODE]: response.statusCode,
+      };
+      if (cLen) {
+        attrs[SemanticAttributes.HTTP_RESPONSE_CONTENT_LENGTH] = cLen;
+      }
+      span.setAttributes(attrs);
+      span.setStatus({
+        code: response.statusCode >= 400 ? SpanStatusCode.ERROR : SpanStatusCode.OK,
+        message: String(response.statusCode),
+      });
+    }
+  }
+
+  onDone({ request }: FetchRequestArguments): void {
+    const span = this.spanFromReq.get(request);
+    if (span !== undefined) {
+      span.end();
+      this.spanFromReq.delete(request);
+    }
+  }
+
+  onError({ request, error }: ErrorFetchRequestArguments): void {
+    const span = this.spanFromReq.get(request);
+    if (span !== undefined) {
+      span.recordException(error);
+      span.setStatus({
+        code: SpanStatusCode.ERROR,
+        message: error.message,
+      });
+      span.end();
+    }
+  }
+}

package/src/telemetry/index.ts

@@ -0,0 +1,69 @@
+import { diag, DiagConsoleLogger, DiagLogLevel } from '@opentelemetry/api';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
+import * as opentelemetry from '@opentelemetry/sdk-node';
+
+import type {
+  DelayLoadServiceStartOptions,
+  RequestLocals,
+  ServiceLocals,
+  ServiceStartOptions,
+} from '../types';
+
+import { getAutoInstrumentations } from './instrumentations';
+
+
+// For troubleshooting, set the log level to DiagLogLevel.DEBUG
+diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);
+
+function getExporter() {
+  if (['production', 'staging'].includes(process.env.APP_ENV || process.env.NODE_ENV || '')) {
+    return new OTLPTraceExporter({
+      url: process.env.OTLP_EXPORTER || 'http://otlp-exporter:4318/v1/traces',
+    });
+  }
+  return new opentelemetry.tracing.ConsoleSpanExporter();
+}
+
+export async function startWithTelemetry<
+  SLocals extends ServiceLocals = ServiceLocals,
+  RLocals extends RequestLocals = RequestLocals,
+>(options: DelayLoadServiceStartOptions) {
+  const sdk = new opentelemetry.NodeSDK({
+    serviceName: options.name,
+    autoDetectResources: true,
+    traceExporter: getExporter(),
+    instrumentations: [getAutoInstrumentations({
+      'opentelemetry-instrumentation-node-18-fetch': {
+        onRequest({ request, span, additionalHeaders }) {
+          // This particular line is "GasBuddy" specific, in that we have a number
+          // of services not yet on OpenTelemetry that look for this header instead.
+          // Putting traceId gives us a "shot in heck" of useful searches.
+          if (!/^correlationid:/m.test(request.headers)) {
+            const ctx = span.spanContext();
+            additionalHeaders.correlationid = ctx.traceId;
+            additionalHeaders.span = ctx.spanId;
+          }
+        },
+      },
+    })],
+  });
+  await sdk.start();
+
+  // eslint-disable-next-line import/no-unresolved
+  const { startApp, listen } = await import('../express-app/app.js');
+  // eslint-disable-next-line import/no-dynamic-require, global-require, @typescript-eslint/no-var-requires
+  const { default: service } = require(options.service);
+  const startOptions: ServiceStartOptions<SLocals> = {
+    ...options,
+    service,
+    locals: { ...options.locals } as Partial<SLocals>,
+  };
+  const app = await startApp<SLocals, RLocals>(startOptions);
+  app.locals.logger.info('OpenTelemetry enabled');
+
+  const server = await listen(app, async () => {
+    await sdk.shutdown();
+    app.locals.logger.info('OpenTelemetry shut down');
+  });
+  return { app, server };
+}

package/src/telemetry/instrumentations.ts

@@ -0,0 +1,54 @@
+import type { Instrumentation } from '@opentelemetry/instrumentation';
+import { AwsInstrumentation } from '@opentelemetry/instrumentation-aws-sdk';
+import { DnsInstrumentation } from '@opentelemetry/instrumentation-dns';
+import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express';
+import { GenericPoolInstrumentation } from '@opentelemetry/instrumentation-generic-pool';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
+import { NetInstrumentation } from '@opentelemetry/instrumentation-net';
+import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';
+import { PinoInstrumentation } from '@opentelemetry/instrumentation-pino';
+
+import { FetchInstrumentation } from './fetchInstrumentation';
+
+const InstrumentationMap = {
+  // Disable this for now because it logs a stupid message
+  // '@opentelemetry/instrumentation-aws-lambda': AwsLambdaInstrumentation,
+  '@opentelemetry/instrumentation-http': HttpInstrumentation,
+  'opentelemetry-instrumentation-node-18-fetch': FetchInstrumentation,
+  '@opentelemetry/instrumentation-aws-sdk': AwsInstrumentation,
+  '@opentelemetry/instrumentation-dns': DnsInstrumentation,
+  '@opentelemetry/instrumentation-express': ExpressInstrumentation,
+  '@opentelemetry/instrumentation-generic-pool': GenericPoolInstrumentation,
+  '@opentelemetry/instrumentation-ioredis': IORedisInstrumentation,
+  '@opentelemetry/instrumentation-net': NetInstrumentation,
+  '@opentelemetry/instrumentation-pg': PgInstrumentation,
+  '@opentelemetry/instrumentation-pino': PinoInstrumentation,
+};
+
+// Config types inferred automatically from the first argument of the constructor
+type ConfigArg<T> = T extends new (...args: infer U) => unknown ? U[0] : never;
+export type InstrumentationConfigMap = {
+  [Name in keyof typeof InstrumentationMap]?: ConfigArg<typeof InstrumentationMap[Name]>;
+};
+
+export function getAutoInstrumentations(
+  inputConfigs: InstrumentationConfigMap = {},
+): Instrumentation[] {
+  const keys = Object.keys(InstrumentationMap) as Array<keyof typeof InstrumentationMap>;
+  return keys
+    .map((name) => {
+      const Instance = InstrumentationMap[name];
+      // Defaults are defined by the instrumentation itself
+      const userConfig = inputConfigs[name] ?? {};
+
+      try {
+        return new Instance(userConfig);
+      } catch (e) {
+        // eslint-disable-next-line no-console
+        console.error(`Failed to load ${name}`, e);
+        return null;
+      }
+    })
+    .filter((i) => !!i) as Instrumentation[];
+}

package/src/telemetry/requestLogger.ts

@@ -0,0 +1,193 @@
+import type {
+  RequestHandler, Request, Response, ErrorRequestHandler,
+} from 'express';
+
+import { ServiceError } from '../error';
+import type { RequestWithApp, ServiceExpress, ServiceLocals } from '../types';
+import type { ServiceHandler } from '../express-app/types';
+
+const LOG_PREFS = Symbol('Logging information');
+
+interface LogPrefs {
+  start: [number, number];
+  logRequests?: boolean;
+  chunks?: Array<Buffer>;
+  logged: boolean;
+}
+
+interface WithLogPrefs {
+  [LOG_PREFS]: LogPrefs;
+}
+
+interface WithIdentifiedSession {
+  session?: {
+    id?: string;
+  };
+}
+
+interface ErrorWithStatus extends Error { status?: number; }
+
+function getBasicInfo(req: Request) {
+  const url = req.originalUrl || req.url;
+
+  const preInfo: Record<string, string> = {
+    url,
+    m: req.method,
+  };
+
+  return preInfo;
+}
+
+function finishLog<SLocals extends ServiceLocals = ServiceLocals>(
+  app: ServiceExpress<SLocals>,
+  error: Error | undefined,
+  req: Request,
+  res: Response,
+) {
+  const prefs = (res.locals as WithLogPrefs)[LOG_PREFS];
+  if (prefs.logged) {
+    // This happens when error handler runs, but onEnd hasn't fired yet. We only log the first one.
+    return;
+  }
+
+  const { logger, service } = app.locals;
+  const hrdur = process.hrtime(prefs.start);
+
+  const dur = hrdur[0] + hrdur[1] / 1000000000;
+  const endLog: Record<string, string | string[] | number | undefined> = {
+    ...getBasicInfo(req),
+    s: (error as ErrorWithStatus)?.status || res.statusCode || 0,
+    dur,
+  };
+
+  if (res.locals.user?.id) {
+    endLog.u = res.locals.user.id;
+  }
+
+  if (error) {
+    endLog.e = error.message;
+    if (!(error instanceof ServiceError) || error.log_stack) {
+      endLog.st = error.stack;
+    }
+  }
+
+  if (prefs.logRequests) {
+    if (Buffer.isBuffer(req.body)) {
+      endLog.b = req.body.toString('base64');
+    } else if (typeof req.body !== 'string') {
+      endLog.b = JSON.stringify(req.body);
+    } else if (req.body) {
+      endLog.b = req.body;
+    }
+  }
+
+  if (prefs.chunks?.length) {
+    const bodyString = Buffer.concat(prefs.chunks).toString('utf8');
+    if (bodyString) {
+      endLog.resBody = bodyString;
+    }
+  }
+
+  service.getLogFields?.(req as RequestWithApp<SLocals>, endLog);
+  logger.info(endLog, 'req');
+}
+
+export function loggerMiddleware<SLocals extends ServiceLocals = ServiceLocals>(
+  app: ServiceExpress<SLocals>,
+  logRequests?: boolean,
+  logResponses?: boolean,
+): RequestHandler {
+  const { logger, service } = app.locals;
+  return function gblogger(req, res, next) {
+    const prefs: LogPrefs = {
+      start: process.hrtime(),
+      logRequests,
+      chunks: logResponses ? [] : undefined,
+      logged: false,
+    };
+
+    (res.locals as WithLogPrefs)[LOG_PREFS] = prefs;
+
+    if (logResponses) {
+      // res is a read-only stream, so the only way to intercept response
+      // data is to monkey-patch.
+      const oldWrite = res.write;
+      const oldEnd = res.end;
+      res.write = ((...args: Parameters<typeof res['write']>) => {
+        if (prefs.chunks) {
+          prefs.chunks.push(Buffer.isBuffer(args[0]) ? args[0] : Buffer.from(args[0]));
+        }
+        return (oldWrite as typeof res['write']).apply(res, args);
+      }) as typeof res['write'];
+      res.end = ((
+        ...args: Parameters<typeof res['end']>
+      ) => {
+        if (args[0] && prefs.chunks) {
+          prefs.chunks.push(Buffer.isBuffer(args[0]) ? args[0] : Buffer.from(args[0]));
+        }
+        return oldEnd.apply(res, args);
+      }) as typeof res['end'];
+    }
+
+    const preLog: Record<string, string | string[] | number | undefined> = {
+      ...getBasicInfo(req),
+      ref: req.headers.referer || undefined,
+      sid: (req as WithIdentifiedSession).session?.id,
+      c: req.headers.correlationid || undefined,
+    };
+    service.getLogFields?.(req as RequestWithApp<SLocals>, preLog);
+    logger.info(preLog, 'pre');
+
+    const logWriter = () => finishLog(app, undefined, req, res);
+    res.on('finish', logWriter);
+    next();
+  };
+}
+
+export function errorHandlerMiddleware<SLocals extends ServiceLocals = ServiceLocals>(
+  app: ServiceExpress<SLocals>,
+  unnest?: boolean,
+  returnError?: boolean,
+) {
+  const gbErrorHandler: ErrorRequestHandler = (error, req, res, next) => {
+    let loggable: Partial<ServiceError> = error;
+    const body = error.response?.body || error.body;
+    if (unnest && body?.domain && body?.code && body?.message) {
+      loggable = {
+        status: error.status,
+        message: body.message,
+        domain: body.domain,
+        code: body.code,
+        display_message: body.display_message,
+      };
+    }
+    // Set the status to error, even if we aren't going to render the error.
+    res.status(loggable.status || 500);
+    if (returnError) {
+      finishLog(app, error, req, res);
+      const prefs = (res.locals as WithLogPrefs)[LOG_PREFS];
+      prefs.logged = true;
+      res.json({
+        code: loggable.code,
+        message: loggable.message,
+        domain: loggable.domain,
+        display_message: loggable.display_message,
+      });
+    } else {
+      next(error);
+    }
+  };
+  return gbErrorHandler;
+}
+
+export function notFoundMiddleware() {
+  const gbNotFoundHandler: ServiceHandler = (req, res, next) => {
+    const error = new ServiceError(req.app, `Cannot ${req.method} ${req.path}`, {
+      status: 404,
+      code: 'NotFound',
+      domain: 'http',
+    });
+    next(error);
+  };
+  return gbNotFoundHandler as RequestHandler;
+}