midline-agent 0.1.9 → 0.2.0

package/src/cli.ts

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+import { MidlineAgent } from "./agent";
+import { ConfigError } from "./config";
+import { proxyAddress, resolveTarget, startMidlineProxy } from "./proxy";
+
+const USAGE = `Usage: midline-agent proxy [--target <url>] [--port <n>] [--host <addr>]
+
+Forwards traffic to a destination API and reports every exchange to Midline.
+
+  --target   Destination API            (TARGET_API_URL, required)
+  --port     Port to listen on          (MIDLINE_PROXY_PORT, default 8080)
+  --host     Interface to bind          (MIDLINE_PROXY_HOST, default 127.0.0.1)
+
+Midline server:
+  MIDLINE_API_KEY      project API key (without it, traffic is forwarded but not reported)
+  MIDLINE_ENDPOINT     default https://api.usemidline.com
+  MIDLINE_CUSTOM_CA    extra CA for the Midline server only (PEM or file path)
+  TARGET_API_CA        extra CA for the destination only (PEM or file path)
+`;
+
+function flag(args: string[], name: string): string | undefined {
+  const index = args.indexOf(`--${name}`);
+  if (index !== -1) return args[index + 1];
+  const inline = args.find((arg) => arg.startsWith(`--${name}=`));
+  return inline?.slice(name.length + 3);
+}
+
+async function main(): Promise<void> {
+  const args = process.argv.slice(2);
+  if (args[0] !== "proxy" || args.includes("--help") || args.includes("-h")) {
+    process.stdout.write(USAGE);
+    process.exitCode = args[0] === "proxy" || args.length === 0 ? 0 : 1;
+    return;
+  }
+
+  const port = flag(args, "port");
+  const target = flag(args, "target") ?? process.env.TARGET_API_URL;
+  // Fail on the destination before the agent starts talking about its own config.
+  resolveTarget(target);
+
+  const agent = MidlineAgent.init({ serviceName: process.env.MIDLINE_SERVICE_NAME });
+  const server = await startMidlineProxy({
+    target,
+    port: port ? Number(port) : undefined,
+    host: flag(args, "host"),
+    agent,
+  });
+
+  process.stdout.write(
+    `midline proxy listening on ${proxyAddress(server)} -> ${server.proxy.target.origin}${server.proxy.target.pathname}` +
+      `${agent.active ? "" : " (monitoring off)"}\n`,
+  );
+
+  let stopping = false;
+  const stop = (signal: string) => {
+    if (stopping) return;
+    stopping = true;
+    process.stdout.write(`midline proxy: ${signal}, draining\n`);
+    server.close();
+    void agent.shutdown(5000).finally(() => process.exit(0));
+  };
+  process.on("SIGINT", () => stop("SIGINT"));
+  process.on("SIGTERM", () => stop("SIGTERM"));
+}
+
+main().catch((err) => {
+  process.stderr.write(`midline proxy: ${err instanceof ConfigError ? err.message : err?.stack ?? err}\n`);
+  process.exit(1);
+});

package/src/config.ts

@@ -0,0 +1,232 @@
+import { readFileSync } from "fs";
+import { X509Certificate } from "crypto";
+import * as tls from "tls";
+import { CaInput, CaptureOptions, MidlineConfig } from "./types";
+
+export const DEFAULT_ENDPOINT = "https://api.usemidline.com";
+export const INGEST_PATH = "/api/api-monitor/events";
+
+/** A setting that cannot work as given. Thrown at construction, never while serving traffic. */
+export class ConfigError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ConfigError";
+  }
+}
+
+export interface ResolvedCapture {
+  headers: boolean;
+  query: boolean;
+  requestBody: boolean;
+  responseBody: boolean;
+  maxBodyBytes: number;
+}
+
+export interface ResolvedConfig {
+  apiKey: string;
+  serviceName?: string;
+  ingestUrl: URL;
+  batchUrl: URL;
+  /** Full trust store for the Midline endpoint, or undefined for Node's default. */
+  ca?: Array<string | Buffer>;
+  hasCustomCa: boolean;
+  environment?: string;
+  host?: string;
+  region?: string;
+  release?: string;
+  redactFields: string[];
+  redactHeaders: string[];
+  capture: ResolvedCapture;
+  onError?: (message: string) => void;
+  debug: boolean;
+  flushIntervalMs: number;
+  timeoutMs: number;
+  connectTimeoutMs: number;
+  maxBatchSize: number;
+  maxQueueSize: number;
+  maxEventBytes: number;
+  maxRetryDelayMs: number;
+}
+
+export function env(name: string): string | undefined {
+  if (typeof process === "undefined" || !process.env) {
+    return undefined;
+  }
+  const value = process.env[name];
+  return value === undefined || value.trim() === "" ? undefined : value.trim();
+}
+
+export function envFlag(name: string): boolean | undefined {
+  const value = env(name)?.toLowerCase();
+  if (value === undefined) return undefined;
+  if (["1", "true", "yes", "on"].includes(value)) return true;
+  if (["0", "false", "no", "off"].includes(value)) return false;
+  return undefined;
+}
+
+export function envInt(name: string): number | undefined {
+  const value = env(name);
+  if (value === undefined) return undefined;
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+export function isLoopback(hostname: string): boolean {
+  const host = hostname.replace(/^\[|\]$/g, "").toLowerCase();
+  return (
+    host === "localhost" ||
+    host.endsWith(".localhost") ||
+    host === "::1" ||
+    /^127(\.\d{1,3}){3}$/.test(host)
+  );
+}
+
+/**
+ * Turns whatever the user configured into the ingest URL.
+ *
+ * `https://api.usemidline.com`, `https://api.usemidline.com/api/api-monitor/events`
+ * and `https://gateway.internal/midline` all work; the last is treated as a prefix.
+ */
+export function resolveIngestUrl(endpoint: string): URL {
+  let url: URL;
+  try {
+    url = new URL(endpoint);
+  } catch {
+    throw new ConfigError(`endpoint "${endpoint}" is not a valid URL`);
+  }
+
+  if (url.protocol !== "https:" && url.protocol !== "http:") {
+    throw new ConfigError(`endpoint must use https:// (got ${url.protocol}//)`);
+  }
+  if (url.protocol === "http:" && !isLoopback(url.hostname)) {
+    throw new ConfigError(
+      `refusing to send the API key in cleartext to ${url.origin} — use https://. ` +
+        "Plain http:// is only accepted for localhost.",
+    );
+  }
+  if (url.username || url.password) {
+    throw new ConfigError("endpoint must not contain credentials; pass the key as apiKey");
+  }
+
+  url.search = "";
+  url.hash = "";
+
+  const path = url.pathname.replace(/\/+$/, "");
+  if (path.endsWith(`${INGEST_PATH}/batch`)) {
+    url.pathname = path.slice(0, -"/batch".length);
+  } else if (path.endsWith(INGEST_PATH)) {
+    url.pathname = path;
+  } else {
+    url.pathname = `${path}${INGEST_PATH}`;
+  }
+  return url;
+}
+
+/**
+ * Loads extra CAs. Every entry must contain at least one parseable certificate, so a
+ * typo in a path fails loudly at startup instead of silently trusting nothing.
+ */
+export function loadCa(input: CaInput | undefined, label: string): Buffer[] | undefined {
+  if (input === undefined || input === null || (Array.isArray(input) && input.length === 0)) {
+    return undefined;
+  }
+
+  const entries = Array.isArray(input) ? input : [input];
+  return entries.map((entry, index) => {
+    let pem: Buffer;
+    if (Buffer.isBuffer(entry)) {
+      pem = entry;
+    } else if (typeof entry === "string" && entry.includes("-----BEGIN")) {
+      pem = Buffer.from(entry);
+    } else if (typeof entry === "string") {
+      try {
+        pem = readFileSync(entry);
+      } catch (err: any) {
+        throw new ConfigError(`${label}: cannot read CA file "${entry}" (${err?.code ?? err?.message})`);
+      }
+    } else {
+      throw new ConfigError(`${label}: entry ${index} must be a PEM string, a Buffer or a file path`);
+    }
+
+    if (!pem.toString("utf8").includes("-----BEGIN CERTIFICATE-----")) {
+      throw new ConfigError(`${label}: entry ${index} does not contain a PEM certificate`);
+    }
+    try {
+      new X509Certificate(pem);
+    } catch {
+      throw new ConfigError(`${label}: entry ${index} is not a parseable certificate`);
+    }
+    return pem;
+  });
+}
+
+/**
+ * Node's `ca` option *replaces* the default trust store. Passing only a private CA
+ * would make every public certificate untrusted, so extra CAs are appended to the
+ * defaults (including NODE_EXTRA_CA_CERTS where the runtime exposes them).
+ */
+export function trustStore(extra: Buffer[] | undefined): Array<string | Buffer> | undefined {
+  if (!extra || extra.length === 0) {
+    return undefined;
+  }
+  const getCACertificates = (tls as any).getCACertificates as
+    | ((type?: string) => string[])
+    | undefined;
+  const defaults = typeof getCACertificates === "function"
+    ? getCACertificates("default")
+    : [...tls.rootCertificates];
+  return [...defaults, ...extra];
+}
+
+function positiveInt(value: number | undefined, fallback: number, min: number, max: number): number {
+  if (value === undefined || !Number.isFinite(value)) {
+    return fallback;
+  }
+  return Math.min(Math.max(Math.floor(value), min), max);
+}
+
+export function resolveCapture(capture: CaptureOptions | undefined): ResolvedCapture {
+  return {
+    headers: capture?.headers === true,
+    query: capture?.query === true,
+    requestBody: capture?.requestBody === true,
+    responseBody: capture?.responseBody === true,
+    maxBodyBytes: positiveInt(capture?.maxBodyBytes, 4096, 0, 1024 * 1024),
+  };
+}
+
+export function resolveConfig(config: MidlineConfig): ResolvedConfig {
+  const apiKey = (config.apiKey ?? env("MIDLINE_API_KEY") ?? "").trim();
+  const ingestUrl = resolveIngestUrl(config.endpoint || env("MIDLINE_ENDPOINT") || DEFAULT_ENDPOINT);
+  const extraCa = loadCa(config.ca ?? env("MIDLINE_CUSTOM_CA"), "ca / MIDLINE_CUSTOM_CA");
+
+  if (extraCa && ingestUrl.protocol !== "https:") {
+    throw new ConfigError("ca / MIDLINE_CUSTOM_CA is set but the endpoint is not https://");
+  }
+
+  return {
+    apiKey,
+    serviceName: config.serviceName ?? env("MIDLINE_SERVICE_NAME"),
+    ingestUrl,
+    batchUrl: new URL(`${ingestUrl.pathname}/batch`, ingestUrl),
+    ca: trustStore(extraCa),
+    hasCustomCa: Boolean(extraCa),
+    environment: config.environment ?? env("MIDLINE_ENVIRONMENT"),
+    host: config.host,
+    region: config.region,
+    release: config.release ?? env("MIDLINE_RELEASE"),
+    redactFields: [...(config.redactFields ?? []), ...(config.maskFields ?? [])],
+    redactHeaders: config.redactHeaders ?? [],
+    capture: resolveCapture(config.capture),
+    onError: config.onError,
+    debug: config.debug ?? envFlag("MIDLINE_DEBUG") ?? false,
+    flushIntervalMs: positiveInt(config.flushIntervalMs, 1500, 50, 60_000),
+    timeoutMs: positiveInt(config.timeoutMs, 10_000, 100, 120_000),
+    connectTimeoutMs: positiveInt(config.connectTimeoutMs, 5000, 50, 60_000),
+    maxBatchSize: positiveInt(config.maxBatchSize, 100, 1, 500),
+    maxQueueSize: positiveInt(config.maxQueueSize, 1000, 1, 100_000),
+    // The server caps a payload at 64 KiB; a larger event could never be accepted.
+    maxEventBytes: positiveInt(config.maxEventBytes, 64 * 1024, 1024, 64 * 1024),
+    maxRetryDelayMs: positiveInt(config.maxRetryDelayMs, 5 * 60_000, 1000, 60 * 60_000),
+  };
+}

package/src/context.ts

@@ -0,0 +1,48 @@
+import { randomUUID } from "crypto";
+import type { IncomingHttpHeaders } from "http";
+
+export interface RequestContext {
+  /** This hop's id. Taken from `x-request-id` when the caller sent a sane one. */
+  requestId: string;
+  /** Shared across every service a request touches. Falls back to the request id. */
+  correlationId: string;
+  traceId?: string;
+  spanId?: string;
+}
+
+const contexts = new WeakMap<object, RequestContext>();
+
+/** Ids come from the network, so only short, printable ones are reused. */
+function sanitizeId(value: string | string[] | undefined): string | undefined {
+  const raw = Array.isArray(value) ? value[0] : value;
+  if (typeof raw !== "string") return undefined;
+  const trimmed = raw.trim();
+  return /^[A-Za-z0-9._:\/+=-]{1,128}$/.test(trimmed) ? trimmed : undefined;
+}
+
+/** W3C trace context: `00-<32 hex trace id>-<16 hex parent id>-<2 hex flags>`. */
+function parseTraceparent(value: string | string[] | undefined): { traceId: string; spanId: string } | undefined {
+  const raw = Array.isArray(value) ? value[0] : value;
+  const match = typeof raw === "string"
+    ? /^[\da-f]{2}-([\da-f]{32})-([\da-f]{16})-[\da-f]{2}$/.exec(raw.trim().toLowerCase())
+    : null;
+  if (!match || /^0+$/.test(match[1]) || /^0+$/.test(match[2])) {
+    return undefined;
+  }
+  return { traceId: match[1], spanId: match[2] };
+}
+
+export function createRequestContext(headers: IncomingHttpHeaders): RequestContext {
+  const requestId = sanitizeId(headers["x-request-id"]) ?? randomUUID();
+  const correlationId = sanitizeId(headers["x-correlation-id"]) ?? requestId;
+  return { requestId, correlationId, ...parseTraceparent(headers["traceparent"]) };
+}
+
+export function setRequestContext(req: object, context: RequestContext): void {
+  contexts.set(req, context);
+}
+
+/** The ids Midline recorded for this request — useful for stamping your own logs. */
+export function getRequestContext(req: object): RequestContext | undefined {
+  return contexts.get(req);
+}

package/src/errorHandler.ts

@@ -1,20 +1,43 @@
-import { Request, Response, NextFunction } from "express";
+import type { IncomingMessage, ServerResponse } from "http";
 import { MidlineAgent } from "./agent";
 import { getBreadcrumbs } from "./breadcrumbs";
+import { getRequestContext } from "./context";
 
-export function midlineErrorHandler() {
-  return (err: any, req: Request, res: Response, next: NextFunction) => {
-    MidlineAgent.addEvent({
-      type: "error",
-      timestamp: new Date().toISOString(),
-      path: req.path,
-      method: req.method,
-      statusCode: err.status || 500,
-      message: err.message,
-      stack: err.stack,
-      breadcrumbs: getBreadcrumbs(),
-    });
+export interface MidlineErrorHandlerOptions {
+  /** Defaults to the agent created by `MidlineAgent.init()`. */
+  agent?: MidlineAgent;
+}
 
+/**
+ * Express error middleware. Records the error, then passes it on unchanged — it
+ * never sends a response itself. Register it after your routes.
+ */
+export function midlineErrorHandler(options: MidlineErrorHandlerOptions = {}) {
+  // Four named parameters: Express recognises error middleware by arity.
+  return function midlineError(err: any, req: IncomingMessage, _res: ServerResponse, next: (err?: unknown) => void): void {
+    try {
+      const agent = options.agent ?? MidlineAgent.current;
+      if (agent?.active) {
+        const context = getRequestContext(req);
+        const status = Number(err?.status ?? err?.statusCode);
+        agent.addEvent({
+          type: "error",
+          path: (req as IncomingMessage & { originalUrl?: string }).originalUrl ?? req.url ?? "/",
+          method: req.method,
+          statusCode: Number.isInteger(status) && status >= 400 && status <= 599 ? status : 500,
+          message: typeof err?.message === "string" ? err.message : String(err),
+          stack: typeof err?.stack === "string" ? err.stack : undefined,
+          breadcrumbs: getBreadcrumbs(),
+          requestId: context?.requestId,
+          correlationId: context?.correlationId,
+          traceId: context?.traceId,
+          spanId: context?.spanId,
+          integration: "express",
+        });
+      }
+    } catch {
+      // Reporting must not replace the application's own error handling.
+    }
     next(err);
   };
 }

package/src/index.ts

@@ -1,5 +1,22 @@
-export { MidlineAgent } from "./agent";
+export { MidlineAgent, SDK_VERSION } from "./agent";
 export { midlineMiddleware } from "./middleware";
+export type { MidlineMiddlewareOptions } from "./middleware";
 export { midlineErrorHandler } from "./errorHandler";
+export type { MidlineErrorHandlerOptions } from "./errorHandler";
+export { createMidlineProxy, startMidlineProxy } from "./proxy";
+export type { MidlineProxy, MidlineProxyOptions, StartProxyOptions } from "./proxy";
 export { addBreadcrumb } from "./breadcrumbs";
-export type { Breadcrumb, MidlineConfig, MidlineEvent } from "./types";
+export { getRequestContext } from "./context";
+export type { RequestContext } from "./context";
+export { ConfigError } from "./config";
+export type {
+  Breadcrumb,
+  CaInput,
+  CapturedMessage,
+  CaptureOptions,
+  EventCategory,
+  EventSeverity,
+  EventType,
+  MidlineConfig,
+  MidlineEvent,
+} from "./types";

package/src/middleware.ts

@@ -1,62 +1,137 @@
-import { Request, Response, NextFunction } from "express";
+import type { IncomingMessage, ServerResponse } from "http";
 import { MidlineAgent } from "./agent";
 import { addBreadcrumb } from "./breadcrumbs";
+import { ResolvedCapture, resolveCapture } from "./config";
+import { createRequestContext, setRequestContext } from "./context";
+import { BodyTap, headerValue, isCompressed } from "./tap";
+import { CaptureOptions } from "./types";
 
-export function midlineMiddleware() {
-  return (req: Request, res: Response, next: NextFunction) => {
-    const start = Date.now();
-    let logged = false;
+export interface MidlineMiddlewareOptions {
+  /** Defaults to the agent created by `MidlineAgent.init()`, looked up per request. */
+  agent?: MidlineAgent;
+  /** Overrides the agent's `capture` settings for requests seen by this middleware. */
+  capture?: CaptureOptions;
+  /** Return true to leave a request out, e.g. health checks. */
+  ignore?: (req: IncomingMessage) => boolean;
+}
+
+/** The Express additions this reads when they exist. Plain `http` requests work too. */
+type RequestLike = IncomingMessage & {
+  originalUrl?: string;
+  ip?: string;
+  body?: unknown;
+  baseUrl?: string;
+  route?: { path?: unknown };
+};
+
+/**
+ * Records every request that passes through. Mount it before your routes.
+ *
+ * It only listens: nothing here reads the request stream, delays `next()`, or
+ * changes the response. Recording happens on `finish`/`close`, after the response
+ * has been handed to the socket.
+ */
+export function midlineMiddleware(options: MidlineMiddlewareOptions = {}) {
+  const captureOverride = options.capture ? resolveCapture(options.capture) : undefined;
+
+  return function midline(req: IncomingMessage, res: ServerResponse, next: (err?: unknown) => void): void {
+    try {
+      observe(req as RequestLike, res, options, captureOverride);
+    } catch {
+      // Monitoring must never be the reason a request fails.
+    }
+    next();
+  };
+}
+
+function observe(
+  req: RequestLike,
+  res: ServerResponse,
+  options: MidlineMiddlewareOptions,
+  captureOverride: ResolvedCapture | undefined,
+): void {
+  const agent = options.agent ?? MidlineAgent.current;
+  if (!agent || !agent.active || options.ignore?.(req)) {
+    return;
+  }
 
-    const logRequest = () => {
-      if (logged) return;
-      logged = true;
+  const capture = captureOverride ?? agent.capture!;
+  const context = createRequestContext(req.headers);
+  setRequestContext(req, context);
 
-      const duration = Date.now() - start;
-      const statusCode = res.statusCode || 200;
+  const started = process.hrtime.bigint();
+  // originalUrl survives router rewrites of req.url; read it now, not at finish.
+  const url = req.originalUrl ?? req.url ?? "/";
+  const responseTap = capture.responseBody ? tapResponse(res, capture.maxBodyBytes) : undefined;
+
+  let recorded = false;
+  const record = () => {
+    if (recorded) return;
+    recorded = true;
+    try {
+      const aborted = !res.writableFinished;
+      const statusCode = aborted && !res.headersSent ? 499 : res.statusCode;
+      const routePath = req.route?.path;
+      const encoding = headerValue(res.getHeader("content-encoding"));
 
       addBreadcrumb({
         type: "http",
-        message: `${req.method} ${req.path} -> ${statusCode}`,
+        message: `${req.method} ${url.split("?")[0]} -> ${statusCode}`,
         timestamp: new Date().toISOString(),
       });
 
-      MidlineAgent.addEvent({
-        type: "request",
-        timestamp: new Date().toISOString(),
-        path: req.path,
+      agent.recordHttp({
+        integration: req.originalUrl !== undefined ? "express" : "node-http",
+        context,
         method: req.method,
-        statusCode: statusCode,
-        duration,
-        ip: req.ip || req.socket.remoteAddress,
-        userAgent: req.headers["user-agent"]
+        url,
+        statusCode,
+        durationMs: Number(process.hrtime.bigint() - started) / 1e6,
+        ip: req.ip ?? req.socket?.remoteAddress,
+        userAgent: headerValue(req.headers["user-agent"]),
+        routeTemplate: typeof routePath === "string" ? `${req.baseUrl ?? ""}${routePath}` : undefined,
+        aborted,
+        capture,
+        request: {
+          headers: capture.headers ? req.headers : undefined,
+          contentType: headerValue(req.headers["content-type"]),
+          // Whatever a body parser already produced. The raw stream is never read here.
+          body: capture.requestBody ? req.body : undefined,
+        },
+        response: {
+          headers: capture.headers ? (res.getHeaders() as Record<string, unknown>) : undefined,
+          contentType: headerValue(res.getHeader("content-type")),
+          body: responseTap && !isCompressed(encoding) ? responseTap.body : undefined,
+          bodyBytes: responseTap?.bytes,
+          truncated: responseTap?.truncated,
+        },
       });
-    };
-
-    // Override multiple response methods to ensure we catch all responses
-    const originalSend = res.send.bind(res);
-    const originalJson = res.json.bind(res);
-    const originalEnd = res.end.bind(res);
+    } catch {
+      // See above.
+    }
+  };
 
-    res.send = function (body?: any) {
-      logRequest();
-      return originalSend(body);
-    };
+  res.once("finish", record);
+  res.once("close", record);
+}
 
-    res.json = function (body?: any) {
-      logRequest();
-      return originalJson(body);
-    };
+/** Wraps write/end with every argument passed through untouched, callbacks included. */
+function tapResponse(res: ServerResponse, limit: number): BodyTap {
+  const tap = new BodyTap(limit);
+  const write = res.write;
+  const end = res.end;
 
-    res.end = function (chunk?: any, encoding?: any) {
-      logRequest();
-      return originalEnd(chunk, encoding);
-    };
+  res.write = function (this: ServerResponse, ...args: any[]) {
+    tap.push(args[0], typeof args[1] === "string" ? args[1] : undefined);
+    return (write as (...a: any[]) => boolean).apply(this, args);
+  } as ServerResponse["write"];
 
-    // Also listen for the 'finish' event as a fallback
-    res.once('finish', () => {
-      logRequest();
-    });
+  res.end = function (this: ServerResponse, ...args: any[]) {
+    if (typeof args[0] !== "function") {
+      tap.push(args[0], typeof args[1] === "string" ? args[1] : undefined);
+    }
+    return (end as (...a: any[]) => ServerResponse).apply(this, args);
+  } as ServerResponse["end"];
 
-    next();
-  };
+  return tap;
 }