midline-agent 0.1.9 → 0.3.0

package/src/redact.ts

@@ -0,0 +1,231 @@
+/**
+ * Redaction happens in the host process, before an event is queued. Whatever is
+ * removed here never reaches a socket, a log line or the Midline server.
+ *
+ * Matching is on a normalised key — lower-cased with punctuation stripped — so
+ * `X-API-Key`, `api_key` and `apiKey` are all the same key.
+ */
+
+export const REDACTED = "[REDACTED]";
+
+/** Substrings: any key containing one of these is sensitive. */
+const SENSITIVE_KEY_PARTS = [
+  "password",
+  "passwd",
+  "passphrase",
+  "secret",
+  "token",
+  "apikey",
+  "accesskey",
+  "privatekey",
+  "authorization",
+  "cookie",
+  "session",
+  "credential",
+  "csrf",
+  "xsrf",
+  "signature",
+  "creditcard",
+  "cardnumber",
+  "cvv",
+  "cvc",
+  "ssn",
+  "socialsecurity",
+];
+
+/** Whole keys only — as substrings these would hit words like "author" or "spinner". */
+const SENSITIVE_KEYS_EXACT = new Set(["auth", "pwd", "pin", "otp", "sid", "jwt", "bearer"]);
+
+/** Always redacted by name, even if a user-supplied list somehow unmatched them. */
+export const DEFAULT_SENSITIVE_HEADERS = [
+  "authorization",
+  "proxy-authorization",
+  "cookie",
+  "set-cookie",
+  "x-api-key",
+  "api-key",
+  "x-auth-token",
+  "x-access-token",
+  "x-refresh-token",
+  "x-csrf-token",
+  "x-xsrf-token",
+  "x-amz-security-token",
+];
+
+const VALUE_PATTERNS: Array<[RegExp, string]> = [
+  [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, REDACTED],
+  [/\b(Bearer|Basic|Digest|Token)\s+[A-Za-z0-9._~+\/=-]{8,}/gi, `$1 ${REDACTED}`],
+  [/\beyJ[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}/g, REDACTED],
+  [/\bak_[A-Fa-f0-9]{16,}\b/g, REDACTED],
+  [/\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}\b/g, REDACTED],
+  [/\bAKIA[0-9A-Z]{16}\b/g, REDACTED],
+  [/(\b[a-z][a-z0-9+.-]*:\/\/)[^\s\/@:]+:[^\s\/@]+@/gi, `$1${REDACTED}@`],
+];
+
+/** `key=value` / `key: value` in query strings, log lines and error messages. */
+const KEY_VALUE_PAIR = /(^|[?&;,\s(\[{])([A-Za-z0-9_.%-]{1,64})(\s*[=:]\s*)("[^"]*"|'[^']*'|[^\s&#,;)\]}]+)/g;
+/** `"key": value` in JSON text, including JSON that was cut off mid-value. */
+const JSON_PAIR = /"([^"\\]{1,100})"\s*:\s*("(?:[^"\\]|\\.)*"?|-?\d+(?:\.\d+)?|true|false|null)/g;
+
+export function normalizeKey(key: string): string {
+  return key.toLowerCase().replace(/[^a-z0-9]/g, "");
+}
+
+export class Redactor {
+  private readonly extraKeys: string[];
+  private readonly headerNames: Set<string>;
+
+  constructor(extraFields: string[] = [], extraHeaders: string[] = []) {
+    this.extraKeys = extraFields.map(normalizeKey).filter(Boolean);
+    this.headerNames = new Set(
+      [...DEFAULT_SENSITIVE_HEADERS, ...extraHeaders].map((name) => name.toLowerCase()),
+    );
+  }
+
+  isSensitiveKey(key: string): boolean {
+    const normalized = normalizeKey(key);
+    if (!normalized) return false;
+    if (SENSITIVE_KEYS_EXACT.has(normalized)) return true;
+    if (SENSITIVE_KEY_PARTS.some((part) => normalized.includes(part))) return true;
+    return this.extraKeys.some((part) => normalized.includes(part));
+  }
+
+  /** Masks credentials embedded in free text: bearer tokens, JWTs, key formats, URL userinfo and query params. */
+  string(value: string, maxLength = 2048): string {
+    let out = value.length > maxLength * 4 ? value.slice(0, maxLength * 4) : value;
+    for (const [pattern, replacement] of VALUE_PATTERNS) {
+      out = out.replace(pattern, replacement);
+    }
+    if (out.includes("=") || out.includes(":")) {
+      out = out.replace(KEY_VALUE_PAIR, (match, sep: string, key: string, delimiter: string) => {
+        let decoded = key;
+        try {
+          decoded = decodeURIComponent(key);
+        } catch {
+          // keep the raw key
+        }
+        return this.isSensitiveKey(decoded) ? `${sep}${key}${delimiter}${REDACTED}` : match;
+      });
+    }
+    if (out.includes('"')) {
+      out = out.replace(JSON_PAIR, (match, key: string) =>
+        this.isSensitiveKey(key) ? `"${key}":"${REDACTED}"` : match,
+      );
+    }
+    // The ellipsis counts toward the limit: servers validate these lengths exactly.
+    return out.length > maxLength ? `${out.slice(0, Math.max(0, maxLength - 1))}…` : out;
+  }
+
+  /** Deep copy with sensitive keys and values masked. Bounded in depth, breadth and string length. */
+  value(input: unknown, depth = 0, seen: WeakSet<object> = new WeakSet()): unknown {
+    if (input === null || input === undefined) return input;
+    if (typeof input === "string") return this.string(input);
+    if (typeof input === "number" || typeof input === "boolean") return input;
+    if (typeof input === "bigint") return input.toString();
+    if (typeof input === "function" || typeof input === "symbol") return undefined;
+    if (input instanceof Date) return Number.isNaN(input.getTime()) ? null : input.toISOString();
+    if (Buffer.isBuffer(input) || ArrayBuffer.isView(input)) {
+      return `[Binary ${(input as ArrayBufferView).byteLength} bytes]`;
+    }
+    if (typeof input !== "object") return undefined;
+
+    if (seen.has(input)) return "[Circular]";
+    if (depth >= 8) return "[Truncated]";
+    seen.add(input);
+
+    if (Array.isArray(input)) {
+      const items = input.slice(0, 100).map((item) => this.value(item, depth + 1, seen));
+      if (input.length > 100) items.push(`[${input.length - 100} more]`);
+      return items;
+    }
+
+    const out: Record<string, unknown> = {};
+    let count = 0;
+    for (const [key, nested] of Object.entries(input as Record<string, unknown>)) {
+      if (count++ >= 200) {
+        out["[truncated]"] = "too many keys";
+        break;
+      }
+      out[key] = this.isSensitiveKey(key) ? REDACTED : this.value(nested, depth + 1, seen);
+    }
+    return out;
+  }
+
+  headers(headers: Record<string, unknown> | undefined): Record<string, string> | undefined {
+    if (!headers) return undefined;
+    const out: Record<string, string> = {};
+    for (const [rawName, rawValue] of Object.entries(headers)) {
+      if (rawValue === undefined) continue;
+      const name = rawName.toLowerCase();
+      if (this.headerNames.has(name) || this.isSensitiveKey(name)) {
+        out[name] = REDACTED;
+        continue;
+      }
+      const joined = Array.isArray(rawValue) ? rawValue.join(", ") : String(rawValue);
+      out[name] = this.string(joined, 1024);
+    }
+    return out;
+  }
+
+  query(search: string): Record<string, string | string[]> | undefined {
+    if (!search) return undefined;
+    const out: Record<string, string | string[]> = {};
+    const params = new URLSearchParams(search.startsWith("?") ? search.slice(1) : search);
+    for (const key of new Set(params.keys())) {
+      const values = params.getAll(key).map((value) =>
+        this.isSensitiveKey(key) ? REDACTED : this.string(value, 512),
+      );
+      out[key] = values.length === 1 ? values[0] : values;
+    }
+    return out;
+  }
+
+  /**
+   * Redacts a captured body. Structured content is parsed and redacted by key;
+   * text that cannot be parsed (usually because it was truncated) gets key/value
+   * pattern masking instead, so a cut-off JSON body still loses its passwords.
+   */
+  body(raw: unknown, contentType: string | undefined, maxBytes: number): { body?: unknown; truncated?: boolean; omitted?: string } {
+    if (raw === undefined || raw === null || maxBytes <= 0) return {};
+
+    const type = (contentType || "").toLowerCase();
+
+    if (typeof raw === "object" && !Buffer.isBuffer(raw)) {
+      return this.fit(this.value(raw), maxBytes);
+    }
+
+    const text = Buffer.isBuffer(raw) ? raw.toString("utf8") : String(raw);
+
+    if (type.includes("json")) {
+      try {
+        return this.fit(this.value(JSON.parse(text)), maxBytes);
+      } catch {
+        return this.cut(this.string(text, text.length), maxBytes);
+      }
+    }
+    if (type.includes("application/x-www-form-urlencoded")) {
+      return this.fit(this.query(text) ?? {}, maxBytes);
+    }
+    if (!type || type.startsWith("text/") || type.includes("xml") || type.includes("graphql")) {
+      return this.cut(this.string(text, maxBytes * 2), maxBytes);
+    }
+    return { omitted: `content-type ${type.split(";")[0]}` };
+  }
+
+  private fit(value: unknown, maxBytes: number): { body?: unknown; truncated?: boolean } {
+    const serialized = JSON.stringify(value) ?? "";
+    if (Buffer.byteLength(serialized) <= maxBytes) {
+      return { body: value };
+    }
+    return this.cut(serialized, maxBytes);
+  }
+
+  private cut(text: string, maxBytes: number): { body?: unknown; truncated?: boolean } {
+    const bytes = Buffer.from(text);
+    if (bytes.length <= maxBytes) {
+      return { body: text };
+    }
+    // Slicing bytes can split a multi-byte character; the replacement char is harmless here.
+    return { body: bytes.subarray(0, maxBytes).toString("utf8"), truncated: true };
+  }
+}

package/src/tap.ts

@@ -0,0 +1,55 @@
+/**
+ * Keeps the first `limit` bytes of a stream that is being written elsewhere, and
+ * counts the rest. Copies what it keeps so a large chunk's backing buffer isn't
+ * retained just because its first few bytes were interesting.
+ */
+export class BodyTap {
+  private readonly chunks: Buffer[] = [];
+  private kept = 0;
+  bytes = 0;
+
+  constructor(private readonly limit: number) {}
+
+  push(chunk: unknown, encoding?: string): void {
+    try {
+      if (chunk === undefined || chunk === null || typeof chunk === "function") return;
+      let buffer: Buffer;
+      if (Buffer.isBuffer(chunk)) {
+        buffer = chunk;
+      } else if (typeof chunk === "string") {
+        buffer = Buffer.from(chunk, Buffer.isEncoding(encoding ?? "") ? (encoding as BufferEncoding) : "utf8");
+      } else if (chunk instanceof Uint8Array) {
+        buffer = Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
+      } else {
+        return;
+      }
+
+      this.bytes += buffer.length;
+      if (this.kept < this.limit) {
+        const part = buffer.subarray(0, this.limit - this.kept);
+        this.chunks.push(Buffer.from(part));
+        this.kept += part.length;
+      }
+    } catch {
+      // Observing a body must never break writing it.
+    }
+  }
+
+  get truncated(): boolean {
+    return this.bytes > this.kept;
+  }
+
+  get body(): Buffer {
+    return Buffer.concat(this.chunks);
+  }
+}
+
+export function headerValue(value: unknown): string | undefined {
+  if (Array.isArray(value)) return value.length ? String(value[0]) : undefined;
+  return value === undefined || value === null ? undefined : String(value);
+}
+
+/** Compressed bytes are unreadable and not worth redacting; the body is skipped instead. */
+export function isCompressed(contentEncoding: string | undefined): boolean {
+  return Boolean(contentEncoding && contentEncoding.toLowerCase() !== "identity");
+}

package/src/transport.ts

@@ -0,0 +1,125 @@
+import * as http from "http";
+import * as https from "https";
+
+export interface PostResult {
+  status: number;
+  headers: http.IncomingHttpHeaders;
+  body: string;
+}
+
+export interface TransportOptions {
+  /** Full trust store for https, or undefined for Node's default. Verification is never disabled. */
+  ca?: Array<string | Buffer>;
+  connectTimeoutMs: number;
+  timeoutMs: number;
+  userAgent: string;
+  /** Response bodies are only read for diagnostics; anything past this is discarded. */
+  maxResponseBytes?: number;
+}
+
+/** A transport failure with a stable `code`, including the two timeouts Node doesn't name. */
+export class TransportError extends Error {
+  constructor(message: string, readonly code: string) {
+    super(message);
+    this.name = "TransportError";
+  }
+}
+
+/**
+ * Minimal JSON POST over Node's own http/https.
+ *
+ * Node's modules rather than fetch because this needs things fetch doesn't expose
+ * portably: a per-endpoint CA that extends rather than replaces the trust store, a
+ * connect timeout separate from the request deadline, and sockets that don't keep
+ * the host process alive.
+ */
+export class Transport {
+  private readonly agent: http.Agent;
+
+  constructor(origin: URL, private readonly options: TransportOptions) {
+    this.agent = origin.protocol === "https:"
+      ? new https.Agent({ keepAlive: true, maxSockets: 4, ca: options.ca })
+      : new http.Agent({ keepAlive: true, maxSockets: 4 });
+  }
+
+  post(url: URL, body: string, headers: Record<string, string>, keepProcessAlive: boolean): Promise<PostResult> {
+    const { connectTimeoutMs, timeoutMs } = this.options;
+    const maxResponseBytes = this.options.maxResponseBytes ?? 64 * 1024;
+    const isHttps = url.protocol === "https:";
+
+    return new Promise<PostResult>((resolve, reject) => {
+      let settled = false;
+      let connectTimer: NodeJS.Timeout | undefined;
+
+      const settle = (fn: () => void) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(deadline);
+        if (connectTimer) clearTimeout(connectTimer);
+        fn();
+      };
+
+      const request = (isHttps ? https : http).request(url, {
+        method: "POST",
+        agent: this.agent,
+        headers: {
+          ...headers,
+          "content-type": "application/json",
+          "content-length": String(Buffer.byteLength(body)),
+          "user-agent": this.options.userAgent,
+        },
+      });
+
+      const deadline = setTimeout(() => {
+        request.destroy(new TransportError(`no response within ${timeoutMs}ms`, "ETIMEDOUT"));
+      }, timeoutMs);
+
+      request.on("socket", (socket) => {
+        if (!keepProcessAlive) {
+          socket.unref();
+        } else {
+          socket.ref();
+        }
+        // A pooled keep-alive socket is already connected; only time fresh ones.
+        // The overall deadline still covers a handshake that stalls after connect.
+        if ((socket as any).connecting) {
+          connectTimer = setTimeout(() => {
+            request.destroy(new TransportError(`connection not established within ${connectTimeoutMs}ms`, "ECONNECT_TIMEOUT"));
+          }, connectTimeoutMs);
+          socket.once(isHttps ? "secureConnect" : "connect", () => clearTimeout(connectTimer));
+        }
+      });
+
+      request.on("response", (response) => {
+        const chunks: Buffer[] = [];
+        let received = 0;
+        response.on("data", (chunk: Buffer) => {
+          if (received < maxResponseBytes) {
+            chunks.push(chunk.subarray(0, maxResponseBytes - received));
+          }
+          received += chunk.length;
+        });
+        response.on("end", () =>
+          settle(() =>
+            resolve({
+              status: response.statusCode ?? 0,
+              headers: response.headers,
+              body: Buffer.concat(chunks).toString("utf8"),
+            }),
+          ),
+        );
+        response.on("error", (err) => settle(() => reject(err)));
+        response.on("aborted", () =>
+          settle(() => reject(new TransportError("response aborted by the server", "ECONNRESET"))),
+        );
+      });
+
+      request.on("error", (err) => settle(() => reject(err)));
+      request.end(body);
+    });
+  }
+
+  destroy(): void {
+    this.agent.destroy();
+  }
+}

package/src/types.ts

@@ -1,32 +1,152 @@
+/** A PEM string, PEM bytes, a path to a PEM file, or several of those. */
+export type CaInput = string | Buffer | Array<string | Buffer>;
+
+export type EventType = "request" | "error" | "security" | "performance" | "custom" | "console";
+export type EventSeverity = "low" | "medium" | "high" | "critical";
+export type EventCategory = "application" | "infrastructure" | "security" | "performance" | "business";
+
+/**
+ * What gets copied off a request/response in addition to method, path, status and
+ * timing. Everything here is off by default: headers, query strings and bodies are
+ * where credentials and personal data live, so capturing them is an explicit
+ * decision. Whatever is captured is redacted in this process before it is queued.
+ */
+export interface CaptureOptions {
+  /** Request and response headers. Default false. */
+  headers?: boolean;
+  /** Query-string parameters. Default false. */
+  query?: boolean;
+  /** Request body — the parsed `req.body` in middleware mode, the raw stream in proxy mode. Default false. */
+  requestBody?: boolean;
+  /** Response body. Default false. */
+  responseBody?: boolean;
+  /** Per-body cap in bytes; anything longer is cut and marked `truncated`. Default 4096. */
+  maxBodyBytes?: number;
+}
+
 export interface MidlineConfig {
-    apiKey: string;
-    serviceName: string;
-    maskFields?: string[];
-    endpoint?: string;
-    environment?: string;
-    host?: string;
-    region?: string;
-    /** Release/version tag — shows up on every event so Midline can tell you which deploy introduced or reintroduced an issue. */
-    release?: string;
-  }
-
-  export interface Breadcrumb {
-    type: string;
-    message: string;
-    timestamp: string;
-  }
-
-  export interface MidlineEvent {
-    type: "request" | "error";
-    timestamp: string;
-    path: string;
-    method?: string;
-    statusCode?: number;
-    duration?: number;
-    message?: string;
-    stack?: string;
-    ip?: string;
-    userAgent?: string;
-    /** Recent requests leading up to this event — only populated on errors. */
-    breadcrumbs?: Breadcrumb[];
-  }
+  /** Project API key from the Midline dashboard. Env fallback: `MIDLINE_API_KEY`. */
+  apiKey?: string;
+  /** Name of the service reporting. Env fallback: `MIDLINE_SERVICE_NAME`. */
+  serviceName?: string;
+
+  /**
+   * The Midline server — the control plane that receives events. Accepts a base URL
+   * (`https://api.usemidline.com`) or the full ingest URL. `https://` is required;
+   * plain `http://` is only accepted for loopback hosts, so the API key never
+   * crosses a network in cleartext. Env fallback: `MIDLINE_ENDPOINT`.
+   *
+   * This is never the API being monitored. See `createMidlineProxy({ target })`.
+   */
+  endpoint?: string;
+
+  /**
+   * Extra certificate authorities to trust for `endpoint` only — for a self-hosted
+   * Midline server behind a private CA. Added to Node's default trust store, never a
+   * replacement for it, and never a way to switch verification off. Env fallback:
+   * `MIDLINE_CUSTOM_CA` (PEM text or a path to a PEM file).
+   */
+  ca?: CaInput;
+
+  environment?: string;
+  host?: string;
+  region?: string;
+  /** Release/version tag — shows up on every event so Midline can tell you which deploy introduced or reintroduced an issue. */
+  release?: string;
+
+  /** Extra field names to redact, on top of the built-in list. Matched case- and punctuation-insensitively. */
+  redactFields?: string[];
+  /** @deprecated Use `redactFields`. Still honoured. */
+  maskFields?: string[];
+  /** Extra header names to redact, on top of the built-in list. */
+  redactHeaders?: string[];
+  /** What to capture beyond method/path/status/timing. Defaults to nothing. */
+  capture?: CaptureOptions;
+  /**
+   * Also send what the process prints — `console.log`, Nest's logger, anything
+   * written to stdout or stderr — as `console` events, one per line, redacted like
+   * any other text. Initialise the agent before creating the app to include its
+   * startup lines. Off by default. Env fallback: `MIDLINE_CAPTURE_CONSOLE`.
+   */
+  captureConsole?: boolean;
+
+  /** Set to false to keep the agent inert. Env fallback: `MIDLINE_ENABLED`. */
+  enabled?: boolean;
+
+  /**
+   * Receives the agent's own diagnostics so they can go to your logger instead of
+   * the console. Set `debug` too if you want them in both places.
+   */
+  onError?: (message: string) => void;
+  /** Emit every diagnostic instead of one per distinct fault. Env fallback: `MIDLINE_DEBUG`. */
+  debug?: boolean;
+
+  /** How often the queue is flushed, in ms. Default 1500. */
+  flushIntervalMs?: number;
+  /** Deadline for a whole delivery request, in ms. Default 10000. */
+  timeoutMs?: number;
+  /** Deadline for the TCP + TLS handshake, in ms. Default 5000. */
+  connectTimeoutMs?: number;
+  /** Events per delivery request. Default 100, max 500. */
+  maxBatchSize?: number;
+  /** Events buffered while the Midline server is unreachable. Oldest are dropped past this. Default 1000. */
+  maxQueueSize?: number;
+  /** Serialised size cap for a single event; larger events lose their captured bodies first. Default and maximum 65536. */
+  maxEventBytes?: number;
+  /** Upper bound on retry backoff, in ms. Default 300000. */
+  maxRetryDelayMs?: number;
+}
+
+export interface Breadcrumb {
+  type: string;
+  message: string;
+  timestamp: string;
+}
+
+export interface CapturedMessage {
+  headers?: Record<string, string>;
+  query?: Record<string, string | string[]>;
+  body?: unknown;
+  /** Bytes seen on the wire (or serialised), before truncation. */
+  bodyBytes?: number;
+  truncated?: boolean;
+  /** Why a body was not captured, e.g. a binary content type. */
+  omitted?: string;
+}
+
+export interface MidlineEvent {
+  type: EventType;
+  /** ISO timestamp. Defaults to the moment the event is added. */
+  timestamp?: string;
+  path: string;
+  method?: string;
+  statusCode?: number;
+  duration?: number;
+  message?: string;
+  stack?: string;
+  ip?: string;
+  userAgent?: string;
+  severity?: EventSeverity;
+  category?: EventCategory;
+  ruleId?: string;
+  threatDetected?: boolean;
+  /** Recent requests leading up to this event — only populated on errors. */
+  breadcrumbs?: Breadcrumb[];
+
+  requestId?: string;
+  correlationId?: string;
+  traceId?: string;
+  spanId?: string;
+  /** Route pattern when the framework exposes one, e.g. `/users/:id`. */
+  routeTemplate?: string;
+
+  request?: CapturedMessage;
+  response?: CapturedMessage;
+  /** The API a proxied request was forwarded to. Only set in proxy mode. */
+  destination?: { url: string };
+  /** Machine-readable failure code, e.g. `ECONNREFUSED` for an unreachable destination. */
+  errorCode?: string;
+  /** Set when the client disconnected before the response finished. */
+  aborted?: boolean;
+  integration?: "express" | "node-http" | "proxy" | "console" | "manual";
+}