midline-agent 0.1.9 → 0.3.0

package/test/middleware.test.js

@@ -0,0 +1,178 @@
+"use strict";
+
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const express = require("express");
+const { MidlineAgent, midlineMiddleware, midlineErrorHandler, getRequestContext } = require("../dist");
+const { REDACTED } = require("../dist/redact");
+const { startCollector, listen, request, closedPort, waitFor } = require("./helpers");
+
+test("express: captured headers, query and bodies are redacted before they leave the process", async () => {
+  const collector = await startCollector();
+  const agent = new MidlineAgent({
+    apiKey: "ak_mw",
+    endpoint: collector.url,
+    flushIntervalMs: 60_000,
+    capture: { headers: true, query: true, requestBody: true, responseBody: true },
+  });
+
+  const app = express();
+  app.use(midlineMiddleware({ agent }));
+  app.use(express.json());
+  app.post("/login", (req, res) => {
+    res.set("set-cookie", "session=abc");
+    res.json({ access_token: "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.c2lnbmF0dXJl", user: req.body.email, requestId: getRequestContext(req).requestId });
+  });
+  const server = await listen(app);
+
+  try {
+    const response = await request(`http://127.0.0.1:${server.port}/login?token=abc&page=2`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        authorization: "Bearer supersecretvalue123",
+        cookie: "sid=1",
+        "x-request-id": "incoming-req-1",
+        "x-correlation-id": "flow-9",
+      },
+      body: JSON.stringify({ email: "ada@example.com", password: "hunter2" }),
+    });
+    assert.equal(response.status, 200);
+    assert.equal(response.json.requestId, "incoming-req-1");
+
+    await waitFor(() => agent.queued === 1);
+    await agent.flush();
+    const [event] = collector.events();
+
+    assert.equal(event.route, "/login");
+    assert.equal(event.method, "POST");
+    assert.equal(event.statusCode, 200);
+    assert.equal(event.metadata.integrationType, "express");
+    assert.equal(event.metadata.requestId, "incoming-req-1");
+    assert.equal(event.metadata.correlationId, "flow-9");
+    assert.equal(event.metadata.routeTemplate, "/login");
+
+    const { request: req, response: res } = event.payload;
+    assert.equal(req.headers.authorization, REDACTED);
+    assert.equal(req.headers.cookie, REDACTED);
+    assert.equal(req.headers["content-type"], "application/json");
+    assert.deepEqual(req.query, { token: REDACTED, page: "2" });
+    assert.deepEqual(req.body, { email: "ada@example.com", password: REDACTED });
+    assert.equal(res.headers["set-cookie"], REDACTED);
+    assert.equal(res.body.access_token, REDACTED);
+    assert.equal(res.body.user, "ada@example.com");
+
+    const serialized = JSON.stringify(collector.requests);
+    for (const secret of ["supersecretvalue123", "hunter2", "sid=1", "session=abc", "eyJhbGciOiJIUzI1NiJ9"]) {
+      assert.ok(!serialized.includes(secret), `${secret} reached the wire`);
+    }
+  } finally {
+    agent.close();
+    await server.close();
+    await collector.close();
+  }
+});
+
+test("express: nothing is captured beyond method/path/status unless asked", async () => {
+  const collector = await startCollector();
+  const agent = new MidlineAgent({ apiKey: "ak_mw", endpoint: collector.url, flushIntervalMs: 60_000 });
+  const app = express();
+  app.use(midlineMiddleware({ agent }));
+  app.use(express.json());
+  app.post("/items", (req, res) => res.status(201).json({ ok: true }));
+  const server = await listen(app);
+  try {
+    await request(`http://127.0.0.1:${server.port}/items?secret=1`, {
+      method: "POST",
+      headers: { "content-type": "application/json", authorization: "Bearer abcdefghijkl" },
+      body: JSON.stringify({ name: "x" }),
+    });
+    await waitFor(() => agent.queued === 1);
+    await agent.flush();
+    const [event] = collector.events();
+    assert.equal(event.statusCode, 201);
+    assert.equal(event.payload, undefined);
+  } finally {
+    agent.close();
+    await server.close();
+    await collector.close();
+  }
+});
+
+test("express: errors are recorded and still reach the app's own handler", async () => {
+  const collector = await startCollector();
+  const agent = new MidlineAgent({ apiKey: "ak_mw", endpoint: collector.url, flushIntervalMs: 60_000 });
+  const app = express();
+  app.use(midlineMiddleware({ agent }));
+  app.get("/boom", () => {
+    throw Object.assign(new Error("db password=hunter2 failed"), { status: 503 });
+  });
+  app.use(midlineErrorHandler({ agent }));
+  app.use((err, req, res, next) => res.status(err.status).json({ handledBy: "app" }));
+  const server = await listen(app);
+  try {
+    const response = await request(`http://127.0.0.1:${server.port}/boom`);
+    assert.equal(response.status, 503);
+    assert.equal(response.json.handledBy, "app");
+
+    await waitFor(() => agent.queued === 2);
+    await agent.flush();
+    const events = collector.events();
+    const error = events.find((event) => event.eventType === "error");
+    assert.equal(error.statusCode, 503);
+    assert.equal(error.payload.error, `db password=${REDACTED} failed`);
+    assert.ok(error.metadata.requestId, "error is tied to the request id");
+    assert.equal(events.find((event) => event.eventType === "request").metadata.requestId, error.metadata.requestId);
+  } finally {
+    agent.close();
+    await server.close();
+    await collector.close();
+  }
+});
+
+test("the host app is unaffected when the Midline server is unreachable or the agent is off", async () => {
+  const port = await closedPort();
+  const agent = new MidlineAgent({ apiKey: "ak_mw", endpoint: `http://127.0.0.1:${port}`, flushIntervalMs: 20, onError: () => {} });
+  const off = new MidlineAgent({ enabled: false });
+
+  const app = express();
+  app.use(midlineMiddleware({ agent }));
+  app.use(midlineMiddleware({ agent: off }));
+  app.get("/health", (req, res) => res.json({ ok: true }));
+  const server = await listen(app);
+  try {
+    for (let i = 0; i < 5; i++) {
+      const response = await request(`http://127.0.0.1:${server.port}/health`);
+      assert.equal(response.status, 200);
+    }
+    await new Promise((resolve) => setTimeout(resolve, 100));
+    assert.ok(agent.active);
+  } finally {
+    agent.close();
+    await server.close();
+  }
+});
+
+test("plain node http servers work with the same middleware", async () => {
+  const collector = await startCollector();
+  const agent = new MidlineAgent({ apiKey: "ak_mw", endpoint: collector.url, flushIntervalMs: 60_000 });
+  const middleware = midlineMiddleware({ agent });
+  const server = await listen((req, res) => {
+    middleware(req, res, () => {
+      res.writeHead(404, { "content-type": "text/plain" });
+      res.end("nope");
+    });
+  });
+  try {
+    await request(`http://127.0.0.1:${server.port}/missing`);
+    await waitFor(() => agent.queued === 1);
+    await agent.flush();
+    const [event] = collector.events();
+    assert.equal(event.statusCode, 404);
+    assert.equal(event.metadata.integrationType, "node-http");
+  } finally {
+    agent.close();
+    await server.close();
+    await collector.close();
+  }
+});

package/test/proxy.test.js

@@ -0,0 +1,274 @@
+"use strict";
+
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const net = require("net");
+const { MidlineAgent, createMidlineProxy, startMidlineProxy, ConfigError } = require("../dist");
+const { REDACTED } = require("../dist/redact");
+const { makeCerts, startCollector, listen, request, closedPort, waitFor } = require("./helpers");
+
+const certs = makeCerts();
+const needsOpenssl = certs ? {} : { skip: "openssl not available" };
+
+/** A destination API that echoes what it received. */
+function echo(req, res) {
+  const chunks = [];
+  req.on("data", (chunk) => chunks.push(chunk));
+  req.on("end", () => {
+    const body = JSON.stringify({
+      method: req.method,
+      url: req.url,
+      headers: req.headers,
+      body: Buffer.concat(chunks).toString("utf8"),
+    });
+    res.writeHead(200, { "content-type": "application/json", connection: "x-hop", "x-hop": "should-not-pass" });
+    res.end(body);
+  });
+}
+
+async function setup({ target, collectorOptions, agentOptions, proxyOptions } = {}) {
+  const collector = await startCollector(collectorOptions);
+  const agent = new MidlineAgent({ apiKey: "ak_proxy", endpoint: collector.url, flushIntervalMs: 60_000, ...agentOptions });
+  const server = await startMidlineProxy({ target, port: 0, agent, onError: () => {}, ...proxyOptions });
+  const base = `http://127.0.0.1:${server.address().port}`;
+  return {
+    collector,
+    agent,
+    base,
+    close: async () => {
+      agent.close();
+      server.closeAllConnections?.();
+      await new Promise((resolve) => server.close(resolve));
+      await collector.close();
+    },
+  };
+}
+
+test("config: the destination is required, validated and never hardcoded", () => {
+  const previous = process.env.TARGET_API_URL;
+  delete process.env.TARGET_API_URL;
+  assert.throws(() => createMidlineProxy({}), /TARGET_API_URL/);
+  assert.throws(() => createMidlineProxy({ target: "ftp://x" }), ConfigError);
+  assert.throws(() => createMidlineProxy({ target: "http://user:pw@x" }), /credentials/);
+  assert.throws(() => createMidlineProxy({ target: "http://localhost:4000", targetCa: "/nope.pem" }), /cannot read CA file/);
+
+  process.env.TARGET_API_URL = "https://staging.example.com/v2/";
+  const proxy = createMidlineProxy({});
+  assert.equal(proxy.target.href, "https://staging.example.com/v2");
+  proxy.close();
+  if (previous === undefined) delete process.env.TARGET_API_URL;
+  else process.env.TARGET_API_URL = previous;
+});
+
+test("forwards to the destination and reports the exchange to Midline separately", async () => {
+  const destination = await listen(echo);
+  const ctx = await setup({
+    target: `http://127.0.0.1:${destination.port}/base`,
+    agentOptions: { capture: { headers: true, query: true, requestBody: true, responseBody: true } },
+  });
+  try {
+    const response = await request(`${ctx.base}/v1/items?page=2&api_key=abc`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        authorization: "Bearer clientsecretvalue",
+        "proxy-authorization": "Basic Zm9vOmJhcg==",
+        "x-request-id": "rid-7",
+      },
+      body: JSON.stringify({ name: "widget", password: "p4ss" }),
+    });
+
+    assert.equal(response.status, 200);
+    const seen = response.json;
+    assert.equal(seen.method, "POST");
+    assert.equal(seen.url, "/base/v1/items?page=2&api_key=abc", "path and query forwarded unchanged under the target prefix");
+    assert.equal(seen.body, JSON.stringify({ name: "widget", password: "p4ss" }), "the destination gets the real body");
+    assert.equal(seen.headers.authorization, "Bearer clientsecretvalue", "the destination gets the real credentials");
+    assert.equal(seen.headers.host, `127.0.0.1:${destination.port}`);
+    assert.equal(seen.headers["proxy-authorization"], undefined, "hop-by-hop headers are not forwarded");
+    assert.equal(seen.headers["x-request-id"], "rid-7");
+    assert.ok(seen.headers["x-forwarded-for"]);
+    assert.equal(response.headers["x-hop"], undefined, "hop-by-hop response headers are stripped");
+
+    await waitFor(() => ctx.agent.queued === 1);
+    await ctx.agent.flush();
+    const [event] = ctx.collector.events();
+    assert.equal(ctx.collector.requests[0].url, "/api/api-monitor/events/batch", "events go to Midline, not the destination");
+    assert.equal(event.route, "/v1/items");
+    assert.equal(event.metadata.integrationType, "proxy");
+    assert.equal(event.metadata.requestId, "rid-7");
+    assert.equal(event.payload.destination.url, `http://127.0.0.1:${destination.port}/base/v1/items`);
+    assert.equal(event.payload.request.headers.authorization, REDACTED);
+    assert.deepEqual(event.payload.request.query, { page: "2", api_key: REDACTED });
+    assert.deepEqual(event.payload.request.body, { name: "widget", password: REDACTED });
+    assert.equal(event.payload.response.body.body.includes("p4ss"), false, "echoed secrets are masked in the captured response too");
+  } finally {
+    await ctx.close();
+    await destination.close();
+  }
+});
+
+test("destination down: 502 to the client, an infrastructure error event to Midline", async () => {
+  const port = await closedPort();
+  const ctx = await setup({ target: `http://127.0.0.1:${port}` });
+  try {
+    const response = await request(`${ctx.base}/orders`);
+    assert.equal(response.status, 502);
+    assert.equal(response.json.error, "Bad Gateway");
+    assert.ok(response.json.requestId);
+    assert.doesNotMatch(response.text, /127\.0\.0\.1|ECONNREFUSED/, "internal details are not leaked to clients");
+
+    await waitFor(() => ctx.agent.queued === 1);
+    await ctx.agent.flush();
+    const [event] = ctx.collector.events();
+    assert.equal(event.eventType, "error");
+    assert.equal(event.statusCode, 502);
+    assert.equal(event.category, "infrastructure");
+    assert.equal(event.payload.code, "ECONNREFUSED");
+  } finally {
+    await ctx.close();
+  }
+});
+
+test("slow destination: 504 after timeoutMs", async () => {
+  const sockets = new Set();
+  const destination = await listen(() => {});
+  destination.server.on("connection", (socket) => sockets.add(socket));
+  const ctx = await setup({ target: `http://127.0.0.1:${destination.port}`, proxyOptions: { timeoutMs: 150 } });
+  try {
+    const response = await request(`${ctx.base}/slow`);
+    assert.equal(response.status, 504);
+  } finally {
+    for (const socket of sockets) socket.destroy();
+    await ctx.close();
+    await destination.close();
+  }
+});
+
+test("retries: idempotent requests that never reached the destination are retried, writes are not", async () => {
+  let attempts = 0;
+  const destination = await listen((req, res) => {
+    attempts += 1;
+    if (attempts === 1) {
+      req.socket.destroy();
+      return;
+    }
+    echo(req, res);
+  });
+  const ctx = await setup({ target: `http://127.0.0.1:${destination.port}`, proxyOptions: { retries: 2 } });
+  try {
+    const ok = await request(`${ctx.base}/flaky`);
+    assert.equal(ok.status, 200);
+    assert.equal(attempts, 2);
+
+    attempts = 0;
+    const write = await request(`${ctx.base}/flaky`, { method: "POST", body: "x", headers: { "content-length": "1" } });
+    assert.equal(write.status, 502);
+    assert.equal(attempts, 1, "a POST that may have been applied is never replayed");
+  } finally {
+    await ctx.close();
+    await destination.close();
+  }
+});
+
+test("request bodies over the limit get 413", async () => {
+  const destination = await listen(echo);
+  const ctx = await setup({ target: `http://127.0.0.1:${destination.port}`, proxyOptions: { maxRequestBodyBytes: 10 } });
+  try {
+    const response = await request(`${ctx.base}/upload`, { method: "POST", body: "x".repeat(100) });
+    assert.equal(response.status, 413);
+  } finally {
+    await ctx.close();
+    await destination.close();
+  }
+});
+
+test("not an open proxy: absolute-form request lines are refused, // paths stay on the target", async () => {
+  const destination = await listen(echo);
+  const ctx = await setup({ target: `http://127.0.0.1:${destination.port}` });
+  try {
+    const raw = await new Promise((resolve, reject) => {
+      const socket = net.connect(Number(new URL(ctx.base).port), "127.0.0.1", () => {
+        socket.write("GET http://169.254.169.254/latest/meta-data HTTP/1.1\r\nHost: 169.254.169.254\r\nConnection: close\r\n\r\n");
+      });
+      let data = "";
+      socket.on("data", (chunk) => (data += chunk));
+      socket.on("end", () => resolve(data));
+      socket.on("error", reject);
+    });
+    assert.match(raw, /^HTTP\/1\.1 400/);
+
+    const sneaky = await request(`${ctx.base}//evil.example/x`);
+    assert.equal(sneaky.status, 200);
+    assert.equal(sneaky.json.url, "//evil.example/x");
+    assert.equal(sneaky.json.headers.host, `127.0.0.1:${destination.port}`);
+  } finally {
+    await ctx.close();
+    await destination.close();
+  }
+});
+
+test("Midline unreachable: traffic still flows through the proxy", async () => {
+  const destination = await listen(echo);
+  const port = await closedPort();
+  const agent = new MidlineAgent({ apiKey: "ak_proxy", endpoint: `http://127.0.0.1:${port}`, flushIntervalMs: 20, onError: () => {} });
+  const server = await startMidlineProxy({ target: `http://127.0.0.1:${destination.port}`, port: 0, agent });
+  try {
+    for (let i = 0; i < 3; i++) {
+      const response = await request(`http://127.0.0.1:${server.address().port}/ping`);
+      assert.equal(response.status, 200);
+    }
+  } finally {
+    agent.close();
+    server.closeAllConnections?.();
+    await new Promise((resolve) => server.close(resolve));
+    await destination.close();
+  }
+});
+
+test("https destination with a private CA: trusted only via targetCa, never by disabling verification", needsOpenssl, async () => {
+  const destination = await listen(echo, certs.trusted);
+  const target = `https://localhost:${destination.port}`;
+
+  const untrusted = await setup({ target });
+  try {
+    const response = await request(`${untrusted.base}/secure`);
+    assert.equal(response.status, 502);
+    await waitFor(() => untrusted.agent.queued === 1);
+    await untrusted.agent.flush();
+    assert.match(untrusted.collector.events()[0].payload.code, /UNABLE_TO_VERIFY_LEAF_SIGNATURE|UNABLE_TO_GET_ISSUER_CERT_LOCALLY|SELF_SIGNED_CERT_IN_CHAIN/);
+  } finally {
+    await untrusted.close();
+  }
+
+  const trusted = await setup({ target, proxyOptions: { targetCa: certs.ca } });
+  try {
+    const response = await request(`${trusted.base}/secure`);
+    assert.equal(response.status, 200);
+    assert.equal(response.json.url, "/secure");
+  } finally {
+    await trusted.close();
+    await destination.close();
+  }
+});
+
+test("the destination CA and the Midline CA are independent", needsOpenssl, async () => {
+  const destination = await listen(echo, certs.trusted);
+  const collector = await startCollector({ tls: certs.selfSigned });
+  const agent = new MidlineAgent({ apiKey: "ak_proxy", endpoint: collector.url, flushIntervalMs: 60_000, onError: () => {} });
+  const server = await startMidlineProxy({ target: `https://localhost:${destination.port}`, targetCa: certs.ca, port: 0, agent });
+  try {
+    const response = await request(`http://127.0.0.1:${server.address().port}/x`);
+    assert.equal(response.status, 200, "destination trusted via targetCa");
+    await waitFor(() => agent.queued === 1);
+    await agent.flush();
+    assert.equal(collector.requests.length, 0, "targetCa did not make the self-signed Midline server trusted");
+    assert.equal(agent.queued, 1);
+  } finally {
+    agent.close();
+    server.closeAllConnections?.();
+    await new Promise((resolve) => server.close(resolve));
+    await collector.close();
+    await destination.close();
+  }
+});

package/test/redact.test.js

@@ -0,0 +1,105 @@
+"use strict";
+
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { Redactor, REDACTED } = require("../dist/redact");
+
+const redactor = new Redactor(["card_number"], ["x-tenant-secret-ish"]);
+
+test("sensitive headers are masked by name, others kept", () => {
+  const headers = redactor.headers({
+    Authorization: "Bearer abc.def.ghi",
+    cookie: "sid=1",
+    "Set-Cookie": ["a=1", "b=2"],
+    "x-api-key": "ak_123",
+    "api-key": "k",
+    "X-CSRF-Token": "t",
+    "proxy-authorization": "Basic Zm9vOmJhcg==",
+    "x-tenant-secret-ish": "custom",
+    "content-type": "application/json",
+    "x-request-id": "abc",
+  });
+  for (const name of ["authorization", "cookie", "set-cookie", "x-api-key", "api-key", "x-csrf-token", "proxy-authorization", "x-tenant-secret-ish"]) {
+    assert.equal(headers[name], REDACTED, name);
+  }
+  assert.equal(headers["content-type"], "application/json");
+  assert.equal(headers["x-request-id"], "abc");
+});
+
+test("keys are matched regardless of case and punctuation, at any depth", () => {
+  const out = redactor.value({
+    user: { name: "Ada", password: "hunter2", Access_Token: "x", refreshToken: "y", client_secret: "z" },
+    api_key: "k",
+    items: [{ cardNumber: "4242", card_number: "4242", cvv: "123" }],
+    author: "kept",
+    auth: "gone",
+    tokenCount: 5,
+  });
+  assert.equal(out.user.name, "Ada");
+  assert.equal(out.user.password, REDACTED);
+  assert.equal(out.user.Access_Token, REDACTED);
+  assert.equal(out.user.refreshToken, REDACTED);
+  assert.equal(out.user.client_secret, REDACTED);
+  assert.equal(out.api_key, REDACTED);
+  assert.equal(out.items[0].cardNumber, REDACTED);
+  assert.equal(out.items[0].card_number, REDACTED);
+  assert.equal(out.items[0].cvv, REDACTED);
+  assert.equal(out.author, "kept");
+  assert.equal(out.auth, REDACTED);
+  assert.equal(out.tokenCount, REDACTED, "errs on the side of masking");
+});
+
+test("credentials inside free text are masked", () => {
+  const jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.c2lnbmF0dXJlLXZhbHVl";
+  const text = [
+    "Authorization: Bearer abcdefghijklmnop",
+    `token ${jwt}`,
+    "key ak_2ada04001f975b0bbbca1d3370006b1bfdc47ec8272d92022afa969f2744046e",
+    "db mongodb+srv://admin:s3cret@cluster0.example.net/app",
+    "/callback?code=1&access_token=xyz&page=2",
+    "stripe sk_live_abcdefghijklmnop",
+  ].join("\n");
+  const out = redactor.string(text, 10_000);
+  assert.doesNotMatch(out, /abcdefghijklmnop/);
+  assert.doesNotMatch(out, /eyJhbGci/);
+  assert.doesNotMatch(out, /2ada04001f97/);
+  assert.doesNotMatch(out, /s3cret/);
+  assert.doesNotMatch(out, /xyz/);
+  assert.match(out, /page=2/);
+  assert.match(out, /cluster0\.example\.net/);
+});
+
+test("query strings: sensitive params masked, others kept", () => {
+  assert.deepEqual(redactor.query("page=2&token=abc&tag=a&tag=b"), { page: "2", token: REDACTED, tag: ["a", "b"] });
+});
+
+test("bodies: JSON parsed and redacted, truncated JSON still masked, binary omitted", () => {
+  const json = redactor.body(Buffer.from(JSON.stringify({ email: "a@b.c", password: "p" })), "application/json", 4096);
+  assert.deepEqual(json.body, { email: "a@b.c", password: REDACTED });
+
+  const cut = redactor.body('{"email":"a@b.c","password":"hunter2","note":"', "application/json", 4096);
+  assert.doesNotMatch(String(cut.body), /hunter2/);
+
+  const long = redactor.body({ data: "x".repeat(5000) }, "application/json", 100);
+  assert.equal(long.truncated, true);
+  assert.ok(Buffer.byteLength(long.body) <= 100);
+
+  const form = redactor.body("user=ada&password=p", "application/x-www-form-urlencoded", 4096);
+  assert.deepEqual(form.body, { user: "ada", password: REDACTED });
+
+  assert.deepEqual(redactor.body(Buffer.from([1, 2, 3]), "application/octet-stream", 4096), { omitted: "content-type application/octet-stream" });
+});
+
+test("hostile shapes are bounded: cycles, depth, width", () => {
+  const cyclic = { a: 1 };
+  cyclic.self = cyclic;
+  assert.equal(redactor.value(cyclic).self, "[Circular]");
+
+  let deep = {};
+  const root = deep;
+  for (let i = 0; i < 50; i++) deep = deep.next = {};
+  assert.match(JSON.stringify(redactor.value(root)), /\[Truncated\]/);
+
+  const wide = Object.fromEntries(Array.from({ length: 500 }, (_, i) => [`k${i}`, i]));
+  assert.ok(Object.keys(redactor.value(wide)).length <= 201);
+});