midline-agent 0.1.8 → 0.2.0

package/src/tap.ts

@@ -0,0 +1,55 @@
+/**
+ * Keeps the first `limit` bytes of a stream that is being written elsewhere, and
+ * counts the rest. Copies what it keeps so a large chunk's backing buffer isn't
+ * retained just because its first few bytes were interesting.
+ */
+export class BodyTap {
+  private readonly chunks: Buffer[] = [];
+  private kept = 0;
+  bytes = 0;
+
+  constructor(private readonly limit: number) {}
+
+  push(chunk: unknown, encoding?: string): void {
+    try {
+      if (chunk === undefined || chunk === null || typeof chunk === "function") return;
+      let buffer: Buffer;
+      if (Buffer.isBuffer(chunk)) {
+        buffer = chunk;
+      } else if (typeof chunk === "string") {
+        buffer = Buffer.from(chunk, Buffer.isEncoding(encoding ?? "") ? (encoding as BufferEncoding) : "utf8");
+      } else if (chunk instanceof Uint8Array) {
+        buffer = Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
+      } else {
+        return;
+      }
+
+      this.bytes += buffer.length;
+      if (this.kept < this.limit) {
+        const part = buffer.subarray(0, this.limit - this.kept);
+        this.chunks.push(Buffer.from(part));
+        this.kept += part.length;
+      }
+    } catch {
+      // Observing a body must never break writing it.
+    }
+  }
+
+  get truncated(): boolean {
+    return this.bytes > this.kept;
+  }
+
+  get body(): Buffer {
+    return Buffer.concat(this.chunks);
+  }
+}
+
+export function headerValue(value: unknown): string | undefined {
+  if (Array.isArray(value)) return value.length ? String(value[0]) : undefined;
+  return value === undefined || value === null ? undefined : String(value);
+}
+
+/** Compressed bytes are unreadable and not worth redacting; the body is skipped instead. */
+export function isCompressed(contentEncoding: string | undefined): boolean {
+  return Boolean(contentEncoding && contentEncoding.toLowerCase() !== "identity");
+}

package/src/transport.ts

@@ -0,0 +1,125 @@
+import * as http from "http";
+import * as https from "https";
+
+export interface PostResult {
+  status: number;
+  headers: http.IncomingHttpHeaders;
+  body: string;
+}
+
+export interface TransportOptions {
+  /** Full trust store for https, or undefined for Node's default. Verification is never disabled. */
+  ca?: Array<string | Buffer>;
+  connectTimeoutMs: number;
+  timeoutMs: number;
+  userAgent: string;
+  /** Response bodies are only read for diagnostics; anything past this is discarded. */
+  maxResponseBytes?: number;
+}
+
+/** A transport failure with a stable `code`, including the two timeouts Node doesn't name. */
+export class TransportError extends Error {
+  constructor(message: string, readonly code: string) {
+    super(message);
+    this.name = "TransportError";
+  }
+}
+
+/**
+ * Minimal JSON POST over Node's own http/https.
+ *
+ * Node's modules rather than fetch because this needs things fetch doesn't expose
+ * portably: a per-endpoint CA that extends rather than replaces the trust store, a
+ * connect timeout separate from the request deadline, and sockets that don't keep
+ * the host process alive.
+ */
+export class Transport {
+  private readonly agent: http.Agent;
+
+  constructor(origin: URL, private readonly options: TransportOptions) {
+    this.agent = origin.protocol === "https:"
+      ? new https.Agent({ keepAlive: true, maxSockets: 4, ca: options.ca })
+      : new http.Agent({ keepAlive: true, maxSockets: 4 });
+  }
+
+  post(url: URL, body: string, headers: Record<string, string>, keepProcessAlive: boolean): Promise<PostResult> {
+    const { connectTimeoutMs, timeoutMs } = this.options;
+    const maxResponseBytes = this.options.maxResponseBytes ?? 64 * 1024;
+    const isHttps = url.protocol === "https:";
+
+    return new Promise<PostResult>((resolve, reject) => {
+      let settled = false;
+      let connectTimer: NodeJS.Timeout | undefined;
+
+      const settle = (fn: () => void) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(deadline);
+        if (connectTimer) clearTimeout(connectTimer);
+        fn();
+      };
+
+      const request = (isHttps ? https : http).request(url, {
+        method: "POST",
+        agent: this.agent,
+        headers: {
+          ...headers,
+          "content-type": "application/json",
+          "content-length": String(Buffer.byteLength(body)),
+          "user-agent": this.options.userAgent,
+        },
+      });
+
+      const deadline = setTimeout(() => {
+        request.destroy(new TransportError(`no response within ${timeoutMs}ms`, "ETIMEDOUT"));
+      }, timeoutMs);
+
+      request.on("socket", (socket) => {
+        if (!keepProcessAlive) {
+          socket.unref();
+        } else {
+          socket.ref();
+        }
+        // A pooled keep-alive socket is already connected; only time fresh ones.
+        // The overall deadline still covers a handshake that stalls after connect.
+        if ((socket as any).connecting) {
+          connectTimer = setTimeout(() => {
+            request.destroy(new TransportError(`connection not established within ${connectTimeoutMs}ms`, "ECONNECT_TIMEOUT"));
+          }, connectTimeoutMs);
+          socket.once(isHttps ? "secureConnect" : "connect", () => clearTimeout(connectTimer));
+        }
+      });
+
+      request.on("response", (response) => {
+        const chunks: Buffer[] = [];
+        let received = 0;
+        response.on("data", (chunk: Buffer) => {
+          if (received < maxResponseBytes) {
+            chunks.push(chunk.subarray(0, maxResponseBytes - received));
+          }
+          received += chunk.length;
+        });
+        response.on("end", () =>
+          settle(() =>
+            resolve({
+              status: response.statusCode ?? 0,
+              headers: response.headers,
+              body: Buffer.concat(chunks).toString("utf8"),
+            }),
+          ),
+        );
+        response.on("error", (err) => settle(() => reject(err)));
+        response.on("aborted", () =>
+          settle(() => reject(new TransportError("response aborted by the server", "ECONNRESET"))),
+        );
+      });
+
+      request.on("error", (err) => settle(() => reject(err)));
+      request.end(body);
+    });
+  }
+
+  destroy(): void {
+    this.agent.destroy();
+  }
+}

package/src/types.ts

@@ -1,32 +1,145 @@
+/** A PEM string, PEM bytes, a path to a PEM file, or several of those. */
+export type CaInput = string | Buffer | Array<string | Buffer>;
+
+export type EventType = "request" | "error" | "security" | "performance" | "custom";
+export type EventSeverity = "low" | "medium" | "high" | "critical";
+export type EventCategory = "application" | "infrastructure" | "security" | "performance" | "business";
+
+/**
+ * What gets copied off a request/response in addition to method, path, status and
+ * timing. Everything here is off by default: headers, query strings and bodies are
+ * where credentials and personal data live, so capturing them is an explicit
+ * decision. Whatever is captured is redacted in this process before it is queued.
+ */
+export interface CaptureOptions {
+  /** Request and response headers. Default false. */
+  headers?: boolean;
+  /** Query-string parameters. Default false. */
+  query?: boolean;
+  /** Request body — the parsed `req.body` in middleware mode, the raw stream in proxy mode. Default false. */
+  requestBody?: boolean;
+  /** Response body. Default false. */
+  responseBody?: boolean;
+  /** Per-body cap in bytes; anything longer is cut and marked `truncated`. Default 4096. */
+  maxBodyBytes?: number;
+}
+
 export interface MidlineConfig {
-    apiKey: string;
-    serviceName: string;
-    maskFields?: string[];
-    endpoint?: string;
-    environment?: string;
-    host?: string;
-    region?: string;
-    /** Release/version tag — shows up on every event so Midline can tell you which deploy introduced or reintroduced an issue. */
-    release?: string;
-  }
-
-  export interface Breadcrumb {
-    type: string;
-    message: string;
-    timestamp: string;
-  }
-
-  export interface MidlineEvent {
-    type: "request" | "error";
-    timestamp: string;
-    path: string;
-    method?: string;
-    statusCode?: number;
-    duration?: number;
-    message?: string;
-    stack?: string;
-    ip?: string;
-    userAgent?: string;
-    /** Recent requests leading up to this event — only populated on errors. */
-    breadcrumbs?: Breadcrumb[];
-  }
+  /** Project API key from the Midline dashboard. Env fallback: `MIDLINE_API_KEY`. */
+  apiKey?: string;
+  /** Name of the service reporting. Env fallback: `MIDLINE_SERVICE_NAME`. */
+  serviceName?: string;
+
+  /**
+   * The Midline server — the control plane that receives events. Accepts a base URL
+   * (`https://api.usemidline.com`) or the full ingest URL. `https://` is required;
+   * plain `http://` is only accepted for loopback hosts, so the API key never
+   * crosses a network in cleartext. Env fallback: `MIDLINE_ENDPOINT`.
+   *
+   * This is never the API being monitored. See `createMidlineProxy({ target })`.
+   */
+  endpoint?: string;
+
+  /**
+   * Extra certificate authorities to trust for `endpoint` only — for a self-hosted
+   * Midline server behind a private CA. Added to Node's default trust store, never a
+   * replacement for it, and never a way to switch verification off. Env fallback:
+   * `MIDLINE_CUSTOM_CA` (PEM text or a path to a PEM file).
+   */
+  ca?: CaInput;
+
+  environment?: string;
+  host?: string;
+  region?: string;
+  /** Release/version tag — shows up on every event so Midline can tell you which deploy introduced or reintroduced an issue. */
+  release?: string;
+
+  /** Extra field names to redact, on top of the built-in list. Matched case- and punctuation-insensitively. */
+  redactFields?: string[];
+  /** @deprecated Use `redactFields`. Still honoured. */
+  maskFields?: string[];
+  /** Extra header names to redact, on top of the built-in list. */
+  redactHeaders?: string[];
+  /** What to capture beyond method/path/status/timing. Defaults to nothing. */
+  capture?: CaptureOptions;
+
+  /** Set to false to keep the agent inert. Env fallback: `MIDLINE_ENABLED`. */
+  enabled?: boolean;
+
+  /**
+   * Receives the agent's own diagnostics so they can go to your logger instead of
+   * the console. Set `debug` too if you want them in both places.
+   */
+  onError?: (message: string) => void;
+  /** Emit every diagnostic instead of one per distinct fault. Env fallback: `MIDLINE_DEBUG`. */
+  debug?: boolean;
+
+  /** How often the queue is flushed, in ms. Default 1500. */
+  flushIntervalMs?: number;
+  /** Deadline for a whole delivery request, in ms. Default 10000. */
+  timeoutMs?: number;
+  /** Deadline for the TCP + TLS handshake, in ms. Default 5000. */
+  connectTimeoutMs?: number;
+  /** Events per delivery request. Default 100, max 500. */
+  maxBatchSize?: number;
+  /** Events buffered while the Midline server is unreachable. Oldest are dropped past this. Default 1000. */
+  maxQueueSize?: number;
+  /** Serialised size cap for a single event; larger events lose their captured bodies first. Default and maximum 65536. */
+  maxEventBytes?: number;
+  /** Upper bound on retry backoff, in ms. Default 300000. */
+  maxRetryDelayMs?: number;
+}
+
+export interface Breadcrumb {
+  type: string;
+  message: string;
+  timestamp: string;
+}
+
+export interface CapturedMessage {
+  headers?: Record<string, string>;
+  query?: Record<string, string | string[]>;
+  body?: unknown;
+  /** Bytes seen on the wire (or serialised), before truncation. */
+  bodyBytes?: number;
+  truncated?: boolean;
+  /** Why a body was not captured, e.g. a binary content type. */
+  omitted?: string;
+}
+
+export interface MidlineEvent {
+  type: EventType;
+  /** ISO timestamp. Defaults to the moment the event is added. */
+  timestamp?: string;
+  path: string;
+  method?: string;
+  statusCode?: number;
+  duration?: number;
+  message?: string;
+  stack?: string;
+  ip?: string;
+  userAgent?: string;
+  severity?: EventSeverity;
+  category?: EventCategory;
+  ruleId?: string;
+  threatDetected?: boolean;
+  /** Recent requests leading up to this event — only populated on errors. */
+  breadcrumbs?: Breadcrumb[];
+
+  requestId?: string;
+  correlationId?: string;
+  traceId?: string;
+  spanId?: string;
+  /** Route pattern when the framework exposes one, e.g. `/users/:id`. */
+  routeTemplate?: string;
+
+  request?: CapturedMessage;
+  response?: CapturedMessage;
+  /** The API a proxied request was forwarded to. Only set in proxy mode. */
+  destination?: { url: string };
+  /** Machine-readable failure code, e.g. `ECONNREFUSED` for an unreachable destination. */
+  errorCode?: string;
+  /** Set when the client disconnected before the response finished. */
+  aborted?: boolean;
+  integration?: "express" | "node-http" | "proxy" | "manual";
+}