@oneuptime/common 12.0.34 → 13.0.1

package/Tests/Server/Services/SecurityEventLastErrorColumnWidth.test.ts

@@ -9,6 +9,7 @@ import { WidenSecurityEventLastErrorColumns1789800000000 } from "../../../Server
 import SchemaMigrations from "../../../Server/Infrastructure/Postgres/SchemaMigrations/Index";
 import DetectionRuleService from "../../../Server/Services/DetectionRuleService";
 import GoogleSecOpsConnectionService from "../../../Server/Services/GoogleSecOpsConnectionService";
+import { REDACTED, redactLogString } from "../../../Server/Utils/LogRedaction";
 import ConnectorErrorMessage, {
   MAX_CONNECTOR_ERROR_MESSAGE_LENGTH,
 } from "../../../Server/Utils/SecurityEvent/ConnectorErrorMessage";
@@ -36,30 +37,49 @@ import { beforeAll, describe, expect, test } from "@jest/globals";
  * varchar(500), and DatabaseService.checkMaxLengthOfFields rejects any
  * string longer than a column's declared max with a BadDataException. The
  * poller's catch block writes { lastPolledAt, lastError } to record a
- * failed poll, and a Google SecOps client error is a ~46 character prefix
- * plus up to 500 characters of echoed response body — 546, past the bound.
- * So the write that was recording the failure became a second failure,
- * threw out of the catch block, and took pollAllDueConnections down with
- * it: nothing stamped, every remaining connection in that tick skipped,
+ * failed poll, and a Google SecOps client error is a short prefix plus an
+ * echo of whatever Google sent back — comfortably past the bound. So the
+ * write that was recording the failure became a second failure, threw out
+ * of the catch block, and took pollAllDueConnections down with it:
+ * nothing stamped, every remaining connection in that tick skipped,
  * repeated every minute forever. The customer's row read lastPolledAt =
  * null AND lastError = null, because the two columns meant to explain the
  * outage were exactly the ones the outage prevented from being written.
  * DetectionRuleEvaluator had the identical pattern on DetectionRule
  * .lastError, where ClickHouse errors echo the whole compiled query.
  *
- * Four things have to line up, and a regression in any one of them brings
- * the silent outage back:
+ * THE CURRENT CONTRACT — and why the old cap is gone. GoogleSecOpsClient
+ * used to clip every echoed body with `.slice(0, BODY_ECHO_LIMIT)`, and an
+ * earlier version of this file leaned on that cap as its bound: read the
+ * number out of the client, assert the stored value was prefix + exactly
+ * that many characters. That cap was deliberately removed. The echo sites
+ * now run the body through the client's own redactErrorBody() (which
+ * decodes nested JSON before handing it to redactLogString /
+ * redactLogValue), which strips credentials but truncates nothing — an
+ * operator gets the WHOLE redacted diagnostic. GoogleSecOpsPoller stores it
+ * with ConnectorErrorMessage.toMessage(error, { truncate: false }), opting
+ * out of the connector clamp entirely, which is only safe because
+ * lastError is unbounded `text`. So the property that makes echoing a
+ * customer-supplied body safe is REDACTION, not length, and the property
+ * that makes storing it safe is that the column has no maximum at all.
+ *
+ * Five things have to line up, and a regression in any one of them brings
+ * the silent outage — or a stored credential — back:
  *   1. both entities declare VeryLongText / `text` with no length, so
  *      getMaxLengthFromTableColumnType returns undefined and the length
  *      check cannot reject a stored error at all,
  *   2. the widest message the producers can actually emit really is
  *      accepted — pinned end to end, from every one of the client's own
  *      failure paths through ConnectorErrorMessage.toMessage to the real
- *      DatabaseService length check,
- *   3. the migration widens the live Postgres columns with ALTER COLUMN
+ *      DatabaseService length check, with the poller's own
+ *      { truncate: false } storing it whole,
+ *   3. every echoed body is redacted, so a private_key / client_secret /
+ *      token in a response body — including one buried in nested JSON
+ *      inside an error message — never reaches the column verbatim,
+ *   4. the migration widens the live Postgres columns with ALTER COLUMN
  *      ... TYPE text rather than DROP + ADD, which would discard every
  *      error already stored, and
- *   4. the migration is registered in SchemaMigrations/Index.ts — an
+ *   5. the migration is registered in SchemaMigrations/Index.ts — an
  *      unregistered migration never runs, so production would stay on
  *      varchar(500) while every entity-level assertion here still passed.
  *      That combination is precisely how this failure comes back.
@@ -92,89 +112,6 @@ const MIGRATIONS_INDEX_SOURCE: string = fs.readFileSync(
   "utf8",
 );
 
-const GOOGLE_SECOPS_CLIENT_SOURCE: string = fs.readFileSync(
-  path.join(
-    __dirname,
-    "..",
-    "..",
-    "..",
-    "Server",
-    "Utils",
-    "SecurityEvent",
-    "GoogleSecOps",
-    "GoogleSecOpsClient.ts",
-  ),
-  "utf8",
-);
-
-/*
- * How many characters of whatever Google sent back the client is willing to
- * echo, read out of the client itself rather than restated here. Restating
- * it would let the client widen its echo without a single test noticing
- * that the stored value grew with it.
- *
- * The premise this file was written on — "there are two error templates and
- * both write `responseText.slice(0, 500)`" — no longer holds. The client
- * reports the failures that arrive on an HTTP 200 as well now (an empty
- * body, a body that is not JSON, a shape the parser does not recognize, a
- * query Chronicle rejected in band), and each of those echoes the thing it
- * could not read through a NAMED constant rather than an inline literal. A
- * scan pinned to the inline form kept passing while it covered less than a
- * third of the sites, which is the failure mode this function exists to
- * prevent. So: every `.slice(0, N)` in the client, named bounds resolved.
- */
-function responseBodyEchoLimits(): Array<number> {
-  const pattern: RegExp = /\.slice\(\s*0\s*,\s*(\w+)\s*\)/g;
-  const limits: Array<number> = [];
-
-  let match: RegExpExecArray | null = pattern.exec(GOOGLE_SECOPS_CLIENT_SOURCE);
-  while (match) {
-    limits.push(resolveNumericToken(String(match[1])));
-    match = pattern.exec(GOOGLE_SECOPS_CLIENT_SOURCE);
-  }
-
-  return limits;
-}
-
-/*
- * An inline bound is its own value; a named one is resolved out of the
- * client's `const NAME: number = N` declaration. An unresolvable name
- * throws rather than becoming NaN and satisfying "they are all equal" by
- * accident.
- */
-const INLINE_BOUND_PATTERN: RegExp = /^\d+$/;
-
-function resolveNumericToken(token: string): number {
-  if (INLINE_BOUND_PATTERN.test(token)) {
-    return Number(token);
-  }
-
-  const declaration: RegExpMatchArray | null =
-    GOOGLE_SECOPS_CLIENT_SOURCE.match(
-      new RegExp(`\\b${token}\\s*:\\s*number\\s*=\\s*(\\d+)`),
-    );
-
-  if (!declaration || declaration[1] === undefined) {
-    throw new Error(
-      `GoogleSecOpsClient truncates with .slice(0, ${token}) but declares no numeric ${token} — this file cannot bound what it stores`,
-    );
-  }
-
-  return Number(declaration[1]);
-}
-
-function responseBodyEchoLimit(): number {
-  const limits: Array<number> = responseBodyEchoLimits();
-
-  if (limits.length === 0) {
-    throw new Error(
-      "GoogleSecOpsClient no longer truncates anything with .slice(0, N) — this file's premise, that every stored error is bounded by a cap the client applies before ConnectorErrorMessage ever sees it, needs revisiting",
-    );
-  }
-
-  return Math.max(...limits);
-}
-
 const { privateKey } = generateKeyPairSync("rsa", {
   modulusLength: 2048,
   privateKeyEncoding: { type: "pkcs8", format: "pem" },
@@ -184,8 +121,8 @@ const { privateKey } = generateKeyPairSync("rsa", {
 /*
  * The real token endpoint, not an example.com stand-in: the client now
  * refuses a token_uri outside Google's hosts, because that URL is
- * customer-supplied and the first 500 characters of whatever answers it are
- * echoed straight into lastError. Nothing here reaches the network — the
+ * customer-supplied and whatever answers it is echoed (redacted, and in
+ * full) straight into lastError. Nothing here reaches the network — the
  * fetch is injected — so the host only has to satisfy that check.
  */
 const SERVICE_ACCOUNT_JSON: string = JSON.stringify({
@@ -198,10 +135,76 @@ const INSTANCE: string =
   "projects/my-project/locations/us/instances/3f0a-instance";
 
 /*
- * Comfortably longer than anything the client will keep, so both templates
- * echo a full-width body and the resulting message is the widest possible.
+ * The client no longer clips what it echoes, so the width of a stored error
+ * is the width of the body Google sent. This is deliberately FAR wider than
+ * MAX_CONNECTOR_ERROR_MESSAGE_LENGTH — four times it — for two reasons:
+ * the resulting message is one the connector clamp WOULD cut if the poller
+ * had not opted out, so the untruncated-storage path is genuinely
+ * exercised rather than trivially satisfied, and it is far past the
+ * varchar(500) bound whose reappearance is the outage this file guards.
+ * It is a local constant on purpose: nothing in the client bounds it any
+ * more, so there is nothing left to read out of the client.
+ */
+const OVERSIZED_RESPONSE_BODY_LENGTH: number =
+  MAX_CONNECTOR_ERROR_MESSAGE_LENGTH * 4;
+
+const OVERSIZED_RESPONSE_BODY: string = "x".repeat(
+  OVERSIZED_RESPONSE_BODY_LENGTH,
+);
+
+/*
+ * Credential-shaped values, in the shapes a token endpoint or a Chronicle
+ * error envelope actually carries them. None of these is real; what matters
+ * is that each one is the sort of value the redactor must never let through
+ * into a column an operator reads in the dashboard.
+ */
+const LEAKED_CLIENT_SECRET: string = "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9a3b";
+const LEAKED_ACCESS_TOKEN: string =
+  "ya29.c.b0Aaekm1J7NotARealTokenValue0123456789";
+const LEAKED_PRIVATE_KEY_BODY: string =
+  "MIIBVgIBADANBgkqhkiG9w0BAQEFAASCAUAwggE8AgEAAkEANotARealKey";
+const LEAKED_PRIVATE_KEY_PEM: string = `-----BEGIN PRIVATE KEY-----\n${LEAKED_PRIVATE_KEY_BODY}\n-----END PRIVATE KEY-----\n`;
+
+const LEAKED_SECRETS: Array<string> = [
+  LEAKED_CLIENT_SECRET,
+  LEAKED_ACCESS_TOKEN,
+  LEAKED_PRIVATE_KEY_BODY,
+];
+
+/*
+ * Credentials sitting in their own fields, which is what an OAuth error
+ * response or a misconfigured proxy echo looks like.
+ */
+const FLAT_CREDENTIAL_BODY: string = JSON.stringify({
+  error: {
+    code: 403,
+    status: "PERMISSION_DENIED",
+    message: "The caller does not have permission.",
+    client_secret: LEAKED_CLIENT_SECRET,
+    access_token: LEAKED_ACCESS_TOKEN,
+    private_key: LEAKED_PRIVATE_KEY_PEM,
+  },
+});
+
+/*
+ * The case commit c0f9e0c2db fixed: the credential JSON is a STRING inside
+ * the error message, so its key quotes arrive escaped. A purely textual
+ * redactor never sees `"client_secret":` in that form and walks straight
+ * past it — which is why redactErrorBody decodes the outer JSON before
+ * redacting. The test below asserts both halves of that: the raw textual
+ * pass really is fooled, and the client's echo is not.
  */
-const OVERSIZED_RESPONSE_BODY: string = "x".repeat(responseBodyEchoLimit() * 4);
+const NESTED_CREDENTIAL_BODY: string = JSON.stringify({
+  error: {
+    code: 400,
+    status: "INVALID_ARGUMENT",
+    message: JSON.stringify({
+      client_secret: LEAKED_CLIENT_SECRET,
+      access_token: LEAKED_ACCESS_TOKEN,
+      private_key: LEAKED_PRIVATE_KEY_PEM,
+    }),
+  },
+});
 
 function makeFetch(
   responses: Array<{ status: number; body: string }>,
@@ -256,6 +259,18 @@ async function thrownClientError(
   throw new Error("Expected GoogleSecOpsClient to throw, but it did not");
 }
 
+/*
+ * Exactly what GoogleSecOpsPoller.ts writes into lastError: the connector
+ * clamp explicitly disabled, then one more redaction pass. Both halves are
+ * reproduced here rather than approximated, because "the value the poller
+ * stores fits the column" is the whole claim of this file.
+ */
+function storedLastError(error: Error): string {
+  return redactLogString(
+    ConnectorErrorMessage.toMessage(error, { truncate: false }),
+  );
+}
+
 interface LastErrorColumn {
   label: string;
   table: string;
@@ -427,12 +442,12 @@ describe("the widest error the producers can emit is storable", () => {
   let widestClientError: Error;
 
   beforeAll(async () => {
-    // Fails at the JWT-bearer token exchange: the first template.
+    // Fails at the JWT-bearer token exchange: the first echo site.
     tokenExchangeError = await thrownClientError([
       { status: 503, body: OVERSIZED_RESPONSE_BODY },
     ]);
 
-    // Token succeeds, the alerts call fails: the second template.
+    // Token succeeds, the alerts call fails: the second echo site.
     alertsFetchError = await thrownClientError([
       successfulTokenResponse(),
       { status: 500, body: OVERSIZED_RESPONSE_BODY },
@@ -444,46 +459,59 @@ describe("the widest error the producers can emit is storable", () => {
         : tokenExchangeError;
   });
 
-  test("every body the client echoes is capped at the same limit", () => {
+  test("each HTTP failure echoes the WHOLE response body, with nothing clipped off the end", () => {
     /*
-     * The premise, restated honestly. It is no longer "exactly two
-     * templates" — the count is whatever the client has, and what must hold
-     * is that all of them truncate at one number. A single site echoing
-     * further than the rest would widen the stored value without widening
-     * anything that bounds it, and the two-site scan this replaced would
-     * not have seen it: four of the seven sites did not exist when that
-     * assertion was written.
+     * What replaced the cap. There is no longer a number to read out of
+     * the client and compare against; the assertion is that the body the
+     * client was handed survives into the message in one piece, tail
+     * included. A `.slice(0, N)` reintroduced at either site fails here,
+     * because the last character of the body would stop being the last
+     * character of the message.
      */
-    const limits: Array<number> = responseBodyEchoLimits();
-
-    expect(limits.length).toBeGreaterThan(2);
-    expect(new Set(limits).size).toBe(1);
-    expect(responseBodyEchoLimit()).toBe(limits[0]);
-  });
-
-  test("each template is a fixed prefix plus a full-width echo of the response body", () => {
-    const echoLimit: number = responseBodyEchoLimit();
-
     for (const error of [tokenExchangeError, alertsFetchError]) {
-      // The tail is the echoed body, clipped to the client's own cap.
-      expect(error.message.endsWith("x".repeat(echoLimit))).toBe(true);
-      expect(error.message.endsWith("x".repeat(echoLimit + 1))).toBe(false);
+      expect(error.message).toContain(OVERSIZED_RESPONSE_BODY);
+      expect(error.message.endsWith(OVERSIZED_RESPONSE_BODY)).toBe(true);
 
       // ...and ahead of it, a non-empty prefix naming the HTTP status.
       const prefix: string = error.message.slice(
         0,
-        error.message.length - echoLimit,
+        error.message.length - OVERSIZED_RESPONSE_BODY_LENGTH,
       );
       expect(prefix.length).toBeGreaterThan(0);
       expect(prefix).toContain("HTTP");
     }
   });
 
+  test("the message width tracks the body width — nothing bounds it but the body itself", async () => {
+    /*
+     * Falsifies the test above from the other side. Hand the same failure
+     * path a body twice as wide and the message grows by exactly the extra
+     * characters: no cap, no ceiling, no rounding. Any truncation
+     * anywhere on this path — in the client, in the redactor, in the
+     * exception — would make the two messages converge instead.
+     */
+    const narrow: Error = await thrownClientError([
+      successfulTokenResponse(),
+      { status: 500, body: OVERSIZED_RESPONSE_BODY },
+    ]);
+    const wide: Error = await thrownClientError([
+      successfulTokenResponse(),
+      { status: 500, body: OVERSIZED_RESPONSE_BODY + OVERSIZED_RESPONSE_BODY },
+    ]);
+
+    expect(wide.message.length - narrow.message.length).toBe(
+      OVERSIZED_RESPONSE_BODY_LENGTH,
+    );
+  });
+
   test("the widest client error overflows the bound lastError used to carry", () => {
     /*
      * This is the string that killed the poller. Prefix + echoed body is
      * larger than varchar(500), so writing it back onto the row raised
-     * BadDataException from inside the catch block.
+     * BadDataException from inside the catch block. It is also past the
+     * connector clamp, which is exactly why the poller opts out of that
+     * clamp and why the column has to be unbounded rather than merely
+     * wider.
      */
     const oldMaxLength: number = getMaxLengthFromTableColumnType(
       TableColumnType.LongText,
@@ -491,31 +519,64 @@ describe("the widest error the producers can emit is storable", () => {
 
     expect(widestClientError.message.length).toBeGreaterThan(oldMaxLength);
 
-    // The overflow is not marginal: the echoed body alone fills the old column.
-    expect(responseBodyEchoLimit()).toBe(oldMaxLength);
+    // The overflow is not marginal: the echoed body alone dwarfs the old column.
+    expect(OVERSIZED_RESPONSE_BODY_LENGTH).toBeGreaterThan(oldMaxLength);
+    expect(widestClientError.message.length).toBeGreaterThan(
+      MAX_CONNECTOR_ERROR_MESSAGE_LENGTH,
+    );
   });
 
-  test("toMessage keeps the widest client error intact and under the connector cap", () => {
-    const stored: string = ConnectorErrorMessage.toMessage(widestClientError);
+  test("the poller's { truncate: false } stores the widest client error whole", () => {
+    /*
+     * GoogleSecOpsPoller writes ConnectorErrorMessage.toMessage(error,
+     * { truncate: false }). The operator is meant to get the complete
+     * redacted diagnostic, so the returned value must be the message
+     * unchanged — no clamp, and specifically no "... (truncated)" marker,
+     * whose presence would mean part of the diagnostic was thrown away.
+     */
+    const stored: string = ConnectorErrorMessage.toMessage(widestClientError, {
+      truncate: false,
+    });
 
-    // Prefix + 500 still fits in 1000, so nothing is lost from an API error.
     expect(stored).toBe(widestClientError.message);
-    expect(stored.length).toBeLessThanOrEqual(
-      MAX_CONNECTOR_ERROR_MESSAGE_LENGTH,
-    );
+    expect(stored).not.toContain("... (truncated)");
+    expect(stored.length).toBeGreaterThan(MAX_CONNECTOR_ERROR_MESSAGE_LENGTH);
+  });
+
+  test("the DEFAULT toMessage still clamps, so the cap stays pinned for callers that use it", () => {
+    /*
+     * Opting out is per call site. Every caller that does NOT pass
+     * { truncate: false } must still be clamped to
+     * MAX_CONNECTOR_ERROR_MESSAGE_LENGTH with the marker, or a narrow
+     * destination somewhere else in the codebase silently starts
+     * overflowing.
+     */
+    for (const error of [
+      tokenExchangeError,
+      alertsFetchError,
+      new Error("y".repeat(MAX_CONNECTOR_ERROR_MESSAGE_LENGTH * 10)),
+    ]) {
+      const clamped: string = ConnectorErrorMessage.toMessage(error);
+
+      expect(clamped.length).toBe(MAX_CONNECTOR_ERROR_MESSAGE_LENGTH);
+      expect(clamped.endsWith("... (truncated)")).toBe(true);
+    }
   });
 
   test.each(LAST_ERROR_COLUMNS)(
-    "$label accepts the widest client error — the real length check does not reject it",
+    "$label accepts the widest client error stored WHOLE — the real length check does not reject it",
     (column: LastErrorColumn) => {
       /*
-       * The end-to-end guard: the same value the poller would store, run
-       * through the same DatabaseService validation that used to throw.
-       * Pre-fix (LongText/varchar(500)) this raised BadDataException, the
-       * throw escaped pollAllDueConnections, and the tick died.
+       * The end-to-end guard: the same value the poller would store —
+       * untruncated and redacted, exactly as GoogleSecOpsPoller builds it
+       * — run through the same DatabaseService validation that used to
+       * throw. Pre-fix (LongText/varchar(500)) this raised
+       * BadDataException, the throw escaped pollAllDueConnections, and the
+       * tick died.
        */
-      const stored: string = ConnectorErrorMessage.toMessage(widestClientError);
+      const stored: string = storedLastError(widestClientError);
 
+      expect(stored.length).toBeGreaterThan(MAX_CONNECTOR_ERROR_MESSAGE_LENGTH);
       expect(() => {
         return column.checkLastError(stored);
       }).not.toThrow();
@@ -531,7 +592,7 @@ describe("the widest error the producers can emit is storable", () => {
        * LongText and it throws exactly as lastError used to. The only
        * thing that changed is the declared column type.
        */
-      const stored: string = ConnectorErrorMessage.toMessage(widestClientError);
+      const stored: string = storedLastError(widestClientError);
 
       expect(
         getMaxLengthFromTableColumnType(
@@ -551,9 +612,10 @@ describe("the widest error the producers can emit is storable", () => {
     (column: LastErrorColumn) => {
       /*
        * The evaluator's side: a ClickHouse error echoes the whole
-       * compiled query, so it has no natural bound. toMessage clamps it
-       * to exactly MAX_CONNECTOR_ERROR_MESSAGE_LENGTH — still double the
-       * old varchar(500), so widening the column is what makes even the
+       * compiled query, so it has no natural bound. That caller keeps the
+       * default clamp, which cuts it to exactly
+       * MAX_CONNECTOR_ERROR_MESSAGE_LENGTH — still double the old
+       * varchar(500), so widening the column is what makes even the
        * clamped value storable.
        */
       const clickhouseError: Error = new Error(
@@ -573,23 +635,6 @@ describe("the widest error the producers can emit is storable", () => {
       }).not.toThrow();
     },
   );
-
-  test("the connector cap is the ceiling on anything that reaches these columns", () => {
-    /*
-     * Ties the clamp to the column: whatever the producers throw, what is
-     * actually written is at most MAX_CONNECTOR_ERROR_MESSAGE_LENGTH, and
-     * a text column has no bound to compare that against.
-     */
-    for (const error of [
-      tokenExchangeError,
-      alertsFetchError,
-      new Error("y".repeat(MAX_CONNECTOR_ERROR_MESSAGE_LENGTH * 10)),
-    ]) {
-      expect(ConnectorErrorMessage.toMessage(error).length).toBeLessThanOrEqual(
-        MAX_CONNECTOR_ERROR_MESSAGE_LENGTH,
-      );
-    }
-  });
 });
 
 interface InBandFailure {
@@ -598,33 +643,44 @@ interface InBandFailure {
   body: string;
   /* Everything the client's message carries ahead of the echoed text. */
   prefix: string;
+  /*
+   * Whether this path echoes the body it could not read. An empty body has
+   * nothing to echo, and a body that is not JSON at all is reported by
+   * kind rather than quoted — so those two are prefix-only, and pinning
+   * that distinction is what stops a future change from quietly turning a
+   * fixed string into an unbounded echo without a test noticing.
+   */
+  echoesBody: boolean;
 }
 
 /*
  * The failures Chronicle reports on an HTTP 200. They are not covered by the
- * two templates above because before the client learned to read the stream
+ * two HTTP paths above because before the client learned to read the stream
  * envelope they did not reach lastError at all — an unreadable body was
  * reported as a healthy poll that found nothing. Each of them now echoes the
- * text it could not read, which is a new way for the stored value to grow,
- * so each has to clear the same end-to-end path: the client's own cap, the
- * connector clamp, and the real length check on both columns.
+ * text it could not read, in full, which is a new way for the stored value
+ * to grow, so each has to clear the same end-to-end path: the poller's
+ * untruncated toMessage and the real length check on both columns.
  */
 const IN_BAND_FAILURES: Array<InBandFailure> = [
   {
     label: "an empty body",
     body: "",
     prefix: "Google SecOps alerts fetch returned an empty body.",
+    echoesBody: false,
   },
   {
     label: "a body that is not JSON",
     body: `<html>${OVERSIZED_RESPONSE_BODY}</html>`,
     prefix: "Google SecOps alerts fetch returned a non-JSON body.",
+    echoesBody: false,
   },
   {
     label: "a shape the parser does not recognize",
     body: JSON.stringify([{ somethingElseEntirely: OVERSIZED_RESPONSE_BODY }]),
     prefix:
       "Google SecOps alerts fetch returned an unrecognized response shape: ",
+    echoesBody: true,
   },
   {
     label: "a query Chronicle rejected in band",
@@ -636,12 +692,13 @@ const IN_BAND_FAILURES: Array<InBandFailure> = [
     ]),
     prefix:
       "Google SecOps alerts query was rejected by Chronicle on an HTTP 200: ",
+    echoesBody: true,
   },
 ];
 
 describe("the failures Chronicle reports on an HTTP 200 are storable too", () => {
   test.each(IN_BAND_FAILURES)(
-    "$label is echoed under the client's own cap and stored whole",
+    "$label is echoed in full and stored whole",
     async (failure: InBandFailure) => {
       const error: Error = await thrownClientError([
         successfulTokenResponse(),
@@ -650,23 +707,154 @@ describe("the failures Chronicle reports on an HTTP 200 are storable too", () =>
 
       expect(error.message.startsWith(failure.prefix)).toBe(true);
 
+      if (failure.echoesBody) {
+        /*
+         * The echo is complete, not clipped — same contract as the HTTP
+         * paths, and the same reason the column has to be unbounded: this
+         * message is already past MAX_CONNECTOR_ERROR_MESSAGE_LENGTH.
+         */
+        expect(error.message).toContain(OVERSIZED_RESPONSE_BODY);
+        expect(error.message.length).toBeGreaterThan(
+          MAX_CONNECTOR_ERROR_MESSAGE_LENGTH,
+        );
+      } else {
+        // Reported by kind, so the body never widens the stored value at all.
+        expect(error.message).toBe(failure.prefix);
+      }
+
       /*
-       * Prefix plus at most one full-width echo — the same shape as the
-       * HTTP templates, so nothing here is wider than the case the column
-       * was widened for.
+       * The poller opts out of the clamp, so lastError carries the whole
+       * thing however wide it is.
        */
-      expect(error.message.length).toBeLessThanOrEqual(
-        failure.prefix.length + responseBodyEchoLimit(),
-      );
-
-      // The clamp leaves it untouched, so lastError carries the whole thing.
-      const stored: string = ConnectorErrorMessage.toMessage(error);
+      const stored: string = ConnectorErrorMessage.toMessage(error, {
+        truncate: false,
+      });
 
       expect(stored).toBe(error.message);
-      expect(stored.length).toBeLessThanOrEqual(
-        MAX_CONNECTOR_ERROR_MESSAGE_LENGTH,
-      );
+      expect(stored).not.toContain("... (truncated)");
+
+      for (const column of LAST_ERROR_COLUMNS) {
+        expect(() => {
+          return column.checkLastError(storedLastError(error));
+        }).not.toThrow();
+      }
+    },
+  );
+});
+
+/*
+ * Redaction is what makes echoing a customer-supplied body safe now that
+ * nothing clips it. The body is attacker-influenced (a token endpoint that
+ * mirrors the request, a proxy that dumps headers) and, worse, routinely
+ * carries the tenant's OWN credentials back — and lastError is rendered in
+ * the dashboard and swept up by support bundles. So the property that used
+ * to be "at most 500 characters reach the column" is now "no credential
+ * reaches the column at all", and it has to hold on every path that echoes.
+ */
+interface RedactedEchoCase {
+  label: string;
+  responses: Array<{ status: number; body: string }>;
+}
+
+const REDACTED_ECHO_CASES: Array<RedactedEchoCase> = [
+  {
+    label: "the token exchange echoing credential fields",
+    responses: [{ status: 400, body: FLAT_CREDENTIAL_BODY }],
+  },
+  {
+    label: "the token exchange echoing credentials nested inside a message",
+    responses: [{ status: 400, body: NESTED_CREDENTIAL_BODY }],
+  },
+  {
+    label: "the alerts fetch echoing credential fields",
+    responses: [
+      successfulTokenResponse(),
+      { status: 403, body: FLAT_CREDENTIAL_BODY },
+    ],
+  },
+  {
+    label: "the alerts fetch echoing credentials nested inside a message",
+    responses: [
+      successfulTokenResponse(),
+      { status: 403, body: NESTED_CREDENTIAL_BODY },
+    ],
+  },
+  {
+    label: "an unrecognized HTTP 200 shape echoing credential fields",
+    responses: [
+      successfulTokenResponse(),
+      {
+        status: 200,
+        body: JSON.stringify({
+          client_secret: LEAKED_CLIENT_SECRET,
+          access_token: LEAKED_ACCESS_TOKEN,
+          private_key: LEAKED_PRIVATE_KEY_PEM,
+        }),
+      },
+    ],
+  },
+  {
+    label: "an in-band query rejection echoing credentials nested in a message",
+    responses: [
+      successfulTokenResponse(),
+      {
+        status: 200,
+        body: JSON.stringify([
+          {
+            validSnapshotQuery: false,
+            queryValidationErrors: [
+              {
+                message: JSON.stringify({
+                  client_secret: LEAKED_CLIENT_SECRET,
+                  access_token: LEAKED_ACCESS_TOKEN,
+                  private_key: LEAKED_PRIVATE_KEY_PEM,
+                }),
+              },
+            ],
+          },
+        ]),
+      },
+    ],
+  },
+];
+
+describe("the echoed body is redacted, which is what replaced the cap", () => {
+  test("a purely textual pass really is fooled by nested credential JSON", () => {
+    /*
+     * The premise behind redactErrorBody decoding the outer JSON first
+     * (commit c0f9e0c2db). Inside a JSON string the credential's key quotes
+     * arrive escaped as \" , which no textual rule keyed on `"key":` can
+     * match — so running redactLogString over the raw body leaves the
+     * secret in the clear. If this ever stops being true the decode step
+     * has become redundant, and the case below stops proving anything.
+     */
+    expect(redactLogString(NESTED_CREDENTIAL_BODY)).toContain(
+      LEAKED_CLIENT_SECRET,
+    );
+  });
+
+  test.each(REDACTED_ECHO_CASES)(
+    "$label never puts the credential in lastError",
+    async (echoCase: RedactedEchoCase) => {
+      const error: Error = await thrownClientError(echoCase.responses);
+      const stored: string = storedLastError(error);
+
+      for (const secret of LEAKED_SECRETS) {
+        expect(error.message).not.toContain(secret);
+        expect(stored).not.toContain(secret);
+      }
+
+      /*
+       * Falsifiable: the body really did reach the message, and the reason
+       * the secret is missing from it is that the redactor replaced the
+       * value — not that the client quietly dropped the whole echo, which
+       * would satisfy "does not contain" while destroying the diagnostic
+       * this column exists to carry.
+       */
+      expect(error.message).toContain(REDACTED);
+      expect(stored).toContain(REDACTED);
 
+      // And the redacted echo is still stored whole, on an unbounded column.
       for (const column of LAST_ERROR_COLUMNS) {
         expect(() => {
           return column.checkLastError(stored);