midline-agent 0.1.9 → 0.3.0

package/src/cli.ts

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+import { MidlineAgent } from "./agent";
+import { ConfigError } from "./config";
+import { proxyAddress, resolveTarget, startMidlineProxy } from "./proxy";
+
+const USAGE = `Usage: midline-agent proxy [--target <url>] [--port <n>] [--host <addr>]
+
+Forwards traffic to a destination API and reports every exchange to Midline.
+
+  --target   Destination API            (TARGET_API_URL, required)
+  --port     Port to listen on          (MIDLINE_PROXY_PORT, default 8080)
+  --host     Interface to bind          (MIDLINE_PROXY_HOST, default 127.0.0.1)
+
+Midline server:
+  MIDLINE_API_KEY      project API key (without it, traffic is forwarded but not reported)
+  MIDLINE_ENDPOINT     default https://api.usemidline.com
+  MIDLINE_CUSTOM_CA    extra CA for the Midline server only (PEM or file path)
+  TARGET_API_CA        extra CA for the destination only (PEM or file path)
+`;
+
+function flag(args: string[], name: string): string | undefined {
+  const index = args.indexOf(`--${name}`);
+  if (index !== -1) return args[index + 1];
+  const inline = args.find((arg) => arg.startsWith(`--${name}=`));
+  return inline?.slice(name.length + 3);
+}
+
+async function main(): Promise<void> {
+  const args = process.argv.slice(2);
+  if (args[0] !== "proxy" || args.includes("--help") || args.includes("-h")) {
+    process.stdout.write(USAGE);
+    process.exitCode = args[0] === "proxy" || args.length === 0 ? 0 : 1;
+    return;
+  }
+
+  const port = flag(args, "port");
+  const target = flag(args, "target") ?? process.env.TARGET_API_URL;
+  // Fail on the destination before the agent starts talking about its own config.
+  resolveTarget(target);
+
+  const agent = MidlineAgent.init({ serviceName: process.env.MIDLINE_SERVICE_NAME });
+  const server = await startMidlineProxy({
+    target,
+    port: port ? Number(port) : undefined,
+    host: flag(args, "host"),
+    agent,
+  });
+
+  process.stdout.write(
+    `midline proxy listening on ${proxyAddress(server)} -> ${server.proxy.target.origin}${server.proxy.target.pathname}` +
+      `${agent.active ? "" : " (monitoring off)"}\n`,
+  );
+
+  let stopping = false;
+  const stop = (signal: string) => {
+    if (stopping) return;
+    stopping = true;
+    process.stdout.write(`midline proxy: ${signal}, draining\n`);
+    server.close();
+    void agent.shutdown(5000).finally(() => process.exit(0));
+  };
+  process.on("SIGINT", () => stop("SIGINT"));
+  process.on("SIGTERM", () => stop("SIGTERM"));
+}
+
+main().catch((err) => {
+  process.stderr.write(`midline proxy: ${err instanceof ConfigError ? err.message : err?.stack ?? err}\n`);
+  process.exit(1);
+});

package/src/config.ts

@@ -0,0 +1,234 @@
+import { readFileSync } from "fs";
+import { X509Certificate } from "crypto";
+import * as tls from "tls";
+import { CaInput, CaptureOptions, MidlineConfig } from "./types";
+
+export const DEFAULT_ENDPOINT = "https://api.usemidline.com";
+export const INGEST_PATH = "/api/api-monitor/events";
+
+/** A setting that cannot work as given. Thrown at construction, never while serving traffic. */
+export class ConfigError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ConfigError";
+  }
+}
+
+export interface ResolvedCapture {
+  headers: boolean;
+  query: boolean;
+  requestBody: boolean;
+  responseBody: boolean;
+  maxBodyBytes: number;
+}
+
+export interface ResolvedConfig {
+  apiKey: string;
+  serviceName?: string;
+  ingestUrl: URL;
+  batchUrl: URL;
+  /** Full trust store for the Midline endpoint, or undefined for Node's default. */
+  ca?: Array<string | Buffer>;
+  hasCustomCa: boolean;
+  environment?: string;
+  host?: string;
+  region?: string;
+  release?: string;
+  redactFields: string[];
+  redactHeaders: string[];
+  capture: ResolvedCapture;
+  captureConsole: boolean;
+  onError?: (message: string) => void;
+  debug: boolean;
+  flushIntervalMs: number;
+  timeoutMs: number;
+  connectTimeoutMs: number;
+  maxBatchSize: number;
+  maxQueueSize: number;
+  maxEventBytes: number;
+  maxRetryDelayMs: number;
+}
+
+export function env(name: string): string | undefined {
+  if (typeof process === "undefined" || !process.env) {
+    return undefined;
+  }
+  const value = process.env[name];
+  return value === undefined || value.trim() === "" ? undefined : value.trim();
+}
+
+export function envFlag(name: string): boolean | undefined {
+  const value = env(name)?.toLowerCase();
+  if (value === undefined) return undefined;
+  if (["1", "true", "yes", "on"].includes(value)) return true;
+  if (["0", "false", "no", "off"].includes(value)) return false;
+  return undefined;
+}
+
+export function envInt(name: string): number | undefined {
+  const value = env(name);
+  if (value === undefined) return undefined;
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+export function isLoopback(hostname: string): boolean {
+  const host = hostname.replace(/^\[|\]$/g, "").toLowerCase();
+  return (
+    host === "localhost" ||
+    host.endsWith(".localhost") ||
+    host === "::1" ||
+    /^127(\.\d{1,3}){3}$/.test(host)
+  );
+}
+
+/**
+ * Turns whatever the user configured into the ingest URL.
+ *
+ * `https://api.usemidline.com`, `https://api.usemidline.com/api/api-monitor/events`
+ * and `https://gateway.internal/midline` all work; the last is treated as a prefix.
+ */
+export function resolveIngestUrl(endpoint: string): URL {
+  let url: URL;
+  try {
+    url = new URL(endpoint);
+  } catch {
+    throw new ConfigError(`endpoint "${endpoint}" is not a valid URL`);
+  }
+
+  if (url.protocol !== "https:" && url.protocol !== "http:") {
+    throw new ConfigError(`endpoint must use https:// (got ${url.protocol}//)`);
+  }
+  if (url.protocol === "http:" && !isLoopback(url.hostname)) {
+    throw new ConfigError(
+      `refusing to send the API key in cleartext to ${url.origin} — use https://. ` +
+        "Plain http:// is only accepted for localhost.",
+    );
+  }
+  if (url.username || url.password) {
+    throw new ConfigError("endpoint must not contain credentials; pass the key as apiKey");
+  }
+
+  url.search = "";
+  url.hash = "";
+
+  const path = url.pathname.replace(/\/+$/, "");
+  if (path.endsWith(`${INGEST_PATH}/batch`)) {
+    url.pathname = path.slice(0, -"/batch".length);
+  } else if (path.endsWith(INGEST_PATH)) {
+    url.pathname = path;
+  } else {
+    url.pathname = `${path}${INGEST_PATH}`;
+  }
+  return url;
+}
+
+/**
+ * Loads extra CAs. Every entry must contain at least one parseable certificate, so a
+ * typo in a path fails loudly at startup instead of silently trusting nothing.
+ */
+export function loadCa(input: CaInput | undefined, label: string): Buffer[] | undefined {
+  if (input === undefined || input === null || (Array.isArray(input) && input.length === 0)) {
+    return undefined;
+  }
+
+  const entries = Array.isArray(input) ? input : [input];
+  return entries.map((entry, index) => {
+    let pem: Buffer;
+    if (Buffer.isBuffer(entry)) {
+      pem = entry;
+    } else if (typeof entry === "string" && entry.includes("-----BEGIN")) {
+      pem = Buffer.from(entry);
+    } else if (typeof entry === "string") {
+      try {
+        pem = readFileSync(entry);
+      } catch (err: any) {
+        throw new ConfigError(`${label}: cannot read CA file "${entry}" (${err?.code ?? err?.message})`);
+      }
+    } else {
+      throw new ConfigError(`${label}: entry ${index} must be a PEM string, a Buffer or a file path`);
+    }
+
+    if (!pem.toString("utf8").includes("-----BEGIN CERTIFICATE-----")) {
+      throw new ConfigError(`${label}: entry ${index} does not contain a PEM certificate`);
+    }
+    try {
+      new X509Certificate(pem);
+    } catch {
+      throw new ConfigError(`${label}: entry ${index} is not a parseable certificate`);
+    }
+    return pem;
+  });
+}
+
+/**
+ * Node's `ca` option *replaces* the default trust store. Passing only a private CA
+ * would make every public certificate untrusted, so extra CAs are appended to the
+ * defaults (including NODE_EXTRA_CA_CERTS where the runtime exposes them).
+ */
+export function trustStore(extra: Buffer[] | undefined): Array<string | Buffer> | undefined {
+  if (!extra || extra.length === 0) {
+    return undefined;
+  }
+  const getCACertificates = (tls as any).getCACertificates as
+    | ((type?: string) => string[])
+    | undefined;
+  const defaults = typeof getCACertificates === "function"
+    ? getCACertificates("default")
+    : [...tls.rootCertificates];
+  return [...defaults, ...extra];
+}
+
+function positiveInt(value: number | undefined, fallback: number, min: number, max: number): number {
+  if (value === undefined || !Number.isFinite(value)) {
+    return fallback;
+  }
+  return Math.min(Math.max(Math.floor(value), min), max);
+}
+
+export function resolveCapture(capture: CaptureOptions | undefined): ResolvedCapture {
+  return {
+    headers: capture?.headers === true,
+    query: capture?.query === true,
+    requestBody: capture?.requestBody === true,
+    responseBody: capture?.responseBody === true,
+    maxBodyBytes: positiveInt(capture?.maxBodyBytes, 4096, 0, 1024 * 1024),
+  };
+}
+
+export function resolveConfig(config: MidlineConfig): ResolvedConfig {
+  const apiKey = (config.apiKey ?? env("MIDLINE_API_KEY") ?? "").trim();
+  const ingestUrl = resolveIngestUrl(config.endpoint || env("MIDLINE_ENDPOINT") || DEFAULT_ENDPOINT);
+  const extraCa = loadCa(config.ca ?? env("MIDLINE_CUSTOM_CA"), "ca / MIDLINE_CUSTOM_CA");
+
+  if (extraCa && ingestUrl.protocol !== "https:") {
+    throw new ConfigError("ca / MIDLINE_CUSTOM_CA is set but the endpoint is not https://");
+  }
+
+  return {
+    apiKey,
+    serviceName: config.serviceName ?? env("MIDLINE_SERVICE_NAME"),
+    ingestUrl,
+    batchUrl: new URL(`${ingestUrl.pathname}/batch`, ingestUrl),
+    ca: trustStore(extraCa),
+    hasCustomCa: Boolean(extraCa),
+    environment: config.environment ?? env("MIDLINE_ENVIRONMENT"),
+    host: config.host,
+    region: config.region,
+    release: config.release ?? env("MIDLINE_RELEASE"),
+    redactFields: [...(config.redactFields ?? []), ...(config.maskFields ?? [])],
+    redactHeaders: config.redactHeaders ?? [],
+    capture: resolveCapture(config.capture),
+    captureConsole: config.captureConsole ?? envFlag("MIDLINE_CAPTURE_CONSOLE") ?? false,
+    onError: config.onError,
+    debug: config.debug ?? envFlag("MIDLINE_DEBUG") ?? false,
+    flushIntervalMs: positiveInt(config.flushIntervalMs, 1500, 50, 60_000),
+    timeoutMs: positiveInt(config.timeoutMs, 10_000, 100, 120_000),
+    connectTimeoutMs: positiveInt(config.connectTimeoutMs, 5000, 50, 60_000),
+    maxBatchSize: positiveInt(config.maxBatchSize, 100, 1, 500),
+    maxQueueSize: positiveInt(config.maxQueueSize, 1000, 1, 100_000),
+    // The server caps a payload at 64 KiB; a larger event could never be accepted.
+    maxEventBytes: positiveInt(config.maxEventBytes, 64 * 1024, 1024, 64 * 1024),
+    maxRetryDelayMs: positiveInt(config.maxRetryDelayMs, 5 * 60_000, 1000, 60 * 60_000),
+  };
+}

package/src/console.ts

@@ -0,0 +1,182 @@
+/**
+ * Console capture: what the host process prints — `console.log`, Nest's logger,
+ * pino, anything that reaches stdout or stderr — copied into Midline line by line.
+ *
+ * The streams are never changed. The original `write` runs first with the
+ * caller's own arguments and its return value is handed straight back, so
+ * back-pressure, callbacks and the terminal output are exactly what they were.
+ */
+
+export type ConsoleStream = "stdout" | "stderr";
+export type ConsoleLevel = "info" | "warn" | "error";
+
+export interface ConsoleLine {
+  stream: ConsoleStream;
+  text: string;
+  level: ConsoleLevel;
+  /** When the line's first chunk was written, not when it was sent. */
+  timestamp: string;
+}
+
+/** Longest line kept. A log line past this is almost always a dumped payload. */
+export const MAX_LINE_CHARS = 4096;
+
+const STREAMS: ConsoleStream[] = ["stdout", "stderr"];
+
+/** CSI (colours, cursor moves), OSC (titles, hyperlinks) and the remaining two-byte escapes. */
+const ANSI_ESCAPE = /\x1B\[[0-?]*[ -\/]*[@-~]|\x1B\][^\x07\x1B]*(?:\x07|\x1B\\)|\x1B[@-Z\\-_]/g;
+
+// Level words are matched in capitals only: lower-case "error" turns up in plenty of harmless lines.
+const ERROR_MARKERS = [/\b(?:ERROR|FATAL|CRITICAL)\b/, /"level"\s*:\s*"?(?:error|fatal|critical|50|60)\b/i];
+const WARN_MARKERS = [/\bWARN(?:ING)?\b/, /"level"\s*:\s*"?(?:warn|warning|40)\b/i];
+
+type WriteFn = (...args: any[]) => boolean;
+
+let suppressed = 0;
+
+/**
+ * Runs `fn` with capture paused. The agent writes its own diagnostics through
+ * this, so a warning about delivery can't become an event that needs delivering.
+ */
+export function withoutConsoleCapture<T>(fn: () => T): T {
+  suppressed += 1;
+  try {
+    return fn();
+  } finally {
+    suppressed -= 1;
+  }
+}
+
+interface Pending {
+  text: string;
+  at: string;
+  /** Already unfinished at the previous flush. */
+  stale: boolean;
+}
+
+export class ConsoleCapture {
+  private readonly originals = new Map<ConsoleStream, WriteFn>();
+  private readonly wrappers = new Map<ConsoleStream, WriteFn>();
+  private readonly pending: Record<ConsoleStream, Pending> = {
+    stdout: { text: "", at: "", stale: false },
+    stderr: { text: "", at: "", stale: false },
+  };
+  private busy = false;
+  private stopped = false;
+
+  constructor(private readonly onLine: (line: ConsoleLine) => void) {}
+
+  install(): void {
+    for (const stream of STREAMS) {
+      const target = process[stream];
+      const original = target.write as WriteFn;
+      const capture = this;
+      const wrapper: WriteFn = function (this: unknown, ...args: any[]) {
+        const result = original.apply(this, args);
+        capture.take(stream, args[0], args[1]);
+        return result;
+      };
+      this.originals.set(stream, original);
+      this.wrappers.set(stream, wrapper);
+      target.write = wrapper as NodeJS.WriteStream["write"];
+    }
+  }
+
+  /** Restores the streams. A wrapper installed on top of ours is left alone — ours just goes quiet underneath it. */
+  uninstall(): void {
+    this.stopped = true;
+    for (const stream of STREAMS) {
+      const wrapper = this.wrappers.get(stream);
+      if (wrapper && process[stream].write === wrapper) {
+        process[stream].write = this.originals.get(stream) as NodeJS.WriteStream["write"];
+      }
+    }
+    this.wrappers.clear();
+    this.originals.clear();
+  }
+
+  /**
+   * Emits lines still waiting for a newline. Without `force`, only the ones that
+   * were already unfinished at the previous call, so a line being written in
+   * pieces isn't cut in half by a timer tick.
+   */
+  flushPending(force = false): void {
+    if (this.stopped || this.busy) return;
+    this.busy = true;
+    try {
+      for (const stream of STREAMS) {
+        const pending = this.pending[stream];
+        if (!pending.text) continue;
+        if (force || pending.stale) {
+          this.pending[stream] = { text: "", at: "", stale: false };
+          this.emit(stream, pending.text, pending.at);
+        } else {
+          pending.stale = true;
+        }
+      }
+    } catch {
+      // Best-effort, like the rest of capture.
+    } finally {
+      this.busy = false;
+    }
+  }
+
+  private take(stream: ConsoleStream, chunk: unknown, encoding: unknown): void {
+    // `busy` also stops a loop if whatever handles a line prints something itself.
+    if (this.stopped || this.busy || suppressed > 0) return;
+    this.busy = true;
+    try {
+      const text = decode(chunk, encoding);
+      if (!text) return;
+
+      const now = new Date().toISOString();
+      const pending = this.pending[stream];
+      const lines = (pending.text + text).split("\n");
+      let rest = lines.pop() ?? "";
+      let at = pending.text ? pending.at : now;
+
+      for (const line of lines) {
+        this.emit(stream, line, at);
+        at = now;
+      }
+      if (rest.length > MAX_LINE_CHARS) {
+        this.emit(stream, rest, at);
+        rest = "";
+      }
+      this.pending[stream] = { text: rest, at: rest ? at : "", stale: false };
+    } catch {
+      // The write itself already happened; a line we couldn't read is just not sent.
+    } finally {
+      this.busy = false;
+    }
+  }
+
+  private emit(stream: ConsoleStream, raw: string, at: string): void {
+    let text = raw.replace(ANSI_ESCAPE, "").replace(/\r$/, "");
+    // A carriage return redraws the line; what's left after the last one is what the terminal shows.
+    text = text.slice(text.lastIndexOf("\r") + 1);
+    if (!text.trim()) return;
+    if (text.length > MAX_LINE_CHARS) text = text.slice(0, MAX_LINE_CHARS);
+    this.onLine({ stream, text, level: levelOf(stream, text), timestamp: at });
+  }
+}
+
+/** Nest, most JSON loggers and console.warn/error leave enough behind to tell a failure from chatter. */
+export function levelOf(stream: ConsoleStream, text: string): ConsoleLevel {
+  const head = text.slice(0, 256);
+  if (ERROR_MARKERS.some((pattern) => pattern.test(head))) return "error";
+  if (WARN_MARKERS.some((pattern) => pattern.test(head))) return "warn";
+  return stream === "stderr" ? "warn" : "info";
+}
+
+function decode(chunk: unknown, encoding: unknown): string {
+  if (typeof chunk === "string") {
+    return typeof encoding === "string" && !/^utf-?8$/i.test(encoding) && Buffer.isEncoding(encoding)
+      ? Buffer.from(chunk, encoding).toString("utf8")
+      : chunk;
+  }
+  if (chunk instanceof Uint8Array) {
+    return Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength).toString("utf8");
+  }
+  return "";
+}

package/src/context.ts

@@ -0,0 +1,48 @@
+import { randomUUID } from "crypto";
+import type { IncomingHttpHeaders } from "http";
+
+export interface RequestContext {
+  /** This hop's id. Taken from `x-request-id` when the caller sent a sane one. */
+  requestId: string;
+  /** Shared across every service a request touches. Falls back to the request id. */
+  correlationId: string;
+  traceId?: string;
+  spanId?: string;
+}
+
+const contexts = new WeakMap<object, RequestContext>();
+
+/** Ids come from the network, so only short, printable ones are reused. */
+function sanitizeId(value: string | string[] | undefined): string | undefined {
+  const raw = Array.isArray(value) ? value[0] : value;
+  if (typeof raw !== "string") return undefined;
+  const trimmed = raw.trim();
+  return /^[A-Za-z0-9._:\/+=-]{1,128}$/.test(trimmed) ? trimmed : undefined;
+}
+
+/** W3C trace context: `00-<32 hex trace id>-<16 hex parent id>-<2 hex flags>`. */
+function parseTraceparent(value: string | string[] | undefined): { traceId: string; spanId: string } | undefined {
+  const raw = Array.isArray(value) ? value[0] : value;
+  const match = typeof raw === "string"
+    ? /^[\da-f]{2}-([\da-f]{32})-([\da-f]{16})-[\da-f]{2}$/.exec(raw.trim().toLowerCase())
+    : null;
+  if (!match || /^0+$/.test(match[1]) || /^0+$/.test(match[2])) {
+    return undefined;
+  }
+  return { traceId: match[1], spanId: match[2] };
+}
+
+export function createRequestContext(headers: IncomingHttpHeaders): RequestContext {
+  const requestId = sanitizeId(headers["x-request-id"]) ?? randomUUID();
+  const correlationId = sanitizeId(headers["x-correlation-id"]) ?? requestId;
+  return { requestId, correlationId, ...parseTraceparent(headers["traceparent"]) };
+}
+
+export function setRequestContext(req: object, context: RequestContext): void {
+  contexts.set(req, context);
+}
+
+/** The ids Midline recorded for this request — useful for stamping your own logs. */
+export function getRequestContext(req: object): RequestContext | undefined {
+  return contexts.get(req);
+}

package/src/errorHandler.ts

@@ -1,20 +1,43 @@
-import { Request, Response, NextFunction } from "express";
+import type { IncomingMessage, ServerResponse } from "http";
 import { MidlineAgent } from "./agent";
 import { getBreadcrumbs } from "./breadcrumbs";
+import { getRequestContext } from "./context";
 
-export function midlineErrorHandler() {
-  return (err: any, req: Request, res: Response, next: NextFunction) => {
-    MidlineAgent.addEvent({
-      type: "error",
-      timestamp: new Date().toISOString(),
-      path: req.path,
-      method: req.method,
-      statusCode: err.status || 500,
-      message: err.message,
-      stack: err.stack,
-      breadcrumbs: getBreadcrumbs(),
-    });
+export interface MidlineErrorHandlerOptions {
+  /** Defaults to the agent created by `MidlineAgent.init()`. */
+  agent?: MidlineAgent;
+}
 
+/**
+ * Express error middleware. Records the error, then passes it on unchanged — it
+ * never sends a response itself. Register it after your routes.
+ */
+export function midlineErrorHandler(options: MidlineErrorHandlerOptions = {}) {
+  // Four named parameters: Express recognises error middleware by arity.
+  return function midlineError(err: any, req: IncomingMessage, _res: ServerResponse, next: (err?: unknown) => void): void {
+    try {
+      const agent = options.agent ?? MidlineAgent.current;
+      if (agent?.active) {
+        const context = getRequestContext(req);
+        const status = Number(err?.status ?? err?.statusCode);
+        agent.addEvent({
+          type: "error",
+          path: (req as IncomingMessage & { originalUrl?: string }).originalUrl ?? req.url ?? "/",
+          method: req.method,
+          statusCode: Number.isInteger(status) && status >= 400 && status <= 599 ? status : 500,
+          message: typeof err?.message === "string" ? err.message : String(err),
+          stack: typeof err?.stack === "string" ? err.stack : undefined,
+          breadcrumbs: getBreadcrumbs(),
+          requestId: context?.requestId,
+          correlationId: context?.correlationId,
+          traceId: context?.traceId,
+          spanId: context?.spanId,
+          integration: "express",
+        });
+      }
+    } catch {
+      // Reporting must not replace the application's own error handling.
+    }
     next(err);
   };
 }

package/src/index.ts

@@ -1,5 +1,22 @@
-export { MidlineAgent } from "./agent";
+export { MidlineAgent, SDK_VERSION } from "./agent";
 export { midlineMiddleware } from "./middleware";
+export type { MidlineMiddlewareOptions } from "./middleware";
 export { midlineErrorHandler } from "./errorHandler";
+export type { MidlineErrorHandlerOptions } from "./errorHandler";
+export { createMidlineProxy, startMidlineProxy } from "./proxy";
+export type { MidlineProxy, MidlineProxyOptions, StartProxyOptions } from "./proxy";
 export { addBreadcrumb } from "./breadcrumbs";
-export type { Breadcrumb, MidlineConfig, MidlineEvent } from "./types";
+export { getRequestContext } from "./context";
+export type { RequestContext } from "./context";
+export { ConfigError } from "./config";
+export type {
+  Breadcrumb,
+  CaInput,
+  CapturedMessage,
+  CaptureOptions,
+  EventCategory,
+  EventSeverity,
+  EventType,
+  MidlineConfig,
+  MidlineEvent,
+} from "./types";