mercury-agent 0.24.3-beta.0 → 0.24.3-beta.1

package/docs/pending-verification.md

@@ -2286,12 +2286,18 @@ The review of the seven 2026-10-02 fixes landed follow-ups on `dev`. Three baked
 
 ## Plan unattended-1003 / S6
 
-The WhatsApp media download now closes its socket when a bound fires, through a Baileys patch (`patches/`). Host-only, nothing baked: a **sync** (which runs `bun install` because `package.json` and `bun.lock` changed) **and a restart**. No image rebuild.
+The WhatsApp media download now closes its socket when a bound fires, through a Baileys patch (`patches/`). *(Superseded 2026-10-07: the patch never reached npm-installed hosts, so it was removed and the media `fetch` is now owned by `fetchDecryptedMedia` — `docs/debug/moderate/2026-10-07-baileys-patch-never-reaches-an-npm-installed-host.md`. The greps below for `signal: options.signal` in `node_modules` now print 0 by design.)* Host-only, nothing baked: a **sync** (which runs `bun install` because `package.json` and `bun.lock` changed) **and a restart**. No image rebuild.
 
 - [x] *(Verified 2026-10-05 on the WSL box, dev `6ddb895`, base `mercury-agent:0.24.2-beta.2` = `63637858e06b`, derived `mercury-agent-ext:b2acae430168` (built 13:36:22 IDT, 482 s), restart 13:28:17 IDT: the sync ran `bun install` and the grep prints `1`.)* MANUAL-VERIFY(2026-10-04): After the sync, `grep -c -F "signal: options.signal" ~/mercury-src/node_modules/@whiskeysockets/baileys/lib/Utils/messages-media.js` prints 1. A 0 means `bun install` did not run or did not apply the patch, and the fix is not deployed even though the code is.
 - [x] *(Verified 2026-10-05 on the WSL box, dev `6ddb895`, base `mercury-agent:0.24.2-beta.2` = `63637858e06b`, derived `mercury-agent-ext:b2acae430168` (built 13:36:22 IDT, 482 s), restart 13:28:17 IDT. Rehearsed, not waited for: the deployed `downloadWhatsAppMedia` from `~/mercury-src`, with production bounds and its patched `node_modules`, against a local raw-TCP server. Open stall: `bound fired while opening` at 30.0 s, `download_failed` to `onSkipped`, server saw the socket close 2 ms later. Body stall after 64 KB: `read bound fired` (idle) at 30.0 s, `download_failed`, socket closed 1 ms later. The live-journal half is still worth a glance the next time a real download stalls.)* MANUAL-VERIFY(2026-10-04): The next time the journal shows `WhatsApp media read bound fired` or `bound fired while opening`, `ss -tnp | grep -c <bun pid>` taken a few seconds later shows no connection to the WhatsApp media host left open for that download, and the user was told the file could not be downloaded.
 - [x] *(Verified 2026-10-05: `docker build -f Dockerfile.host` from `~/mercury-src` at `6ddb895` exited 0, `COPY patches/` ran before `bun install --production`, and the built image's `node_modules/@whiskeysockets/baileys/lib/Utils/messages-media.js` has `signal: options.signal` (1). Throwaway tag removed.)* MANUAL-VERIFY(2026-10-04): If any box builds `Dockerfile.host`, its next build passes the `bun install --production` step (it now copies `patches/` first; without it the install fails with `Couldn't find patch file`).
 
+## baileys-patch-never-reaches-an-npm-installed-host (2026-10-07)
+
+The media download fetches and decrypts itself instead of through a patched Baileys. Host-only: a sync (`bun install`, since `package.json` and `bun.lock` changed) and a restart; no image rebuild.
+
+- [ ] MANUAL-VERIFY(2026-10-07): After deploying, send the bot a photo, a voice note and a PDF in a group it answers in, and confirm it describes each one — real WhatsApp media must decrypt through the new path, which the tests only exercise with media they encrypted themselves.
+
 ## Plan unattended-1003 / S7
 
 `pinchtab_nav` and the session injector now ship as files in the pinchtab extension's `bin/`. They reach the box through the installed snapshot, which is refreshed on restart, and from there into the **derived** image, whose hash includes the bin files. So this is a restart, then wait for the journal's `Built derived agent image` line. No base rebuild.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mercury-agent",
-  "version": "0.24.3-beta.0",
+  "version": "0.24.3-beta.1",
   "description": "Personal AI assistant for chat platforms (WhatsApp, Slack, Discord, Telegram)",
   "license": "MIT",
   "author": "Avishai Tsabari",
@@ -151,8 +151,5 @@
     "ws": ">=8.20.1",
     "qs": ">=6.15.2",
     "lodash": ">=4.18.1"
-  },
-  "patchedDependencies": {
-    "@whiskeysockets/baileys@7.0.0-rc14": "patches/@whiskeysockets%2Fbaileys@7.0.0-rc14.patch"
   }
 }

package/src/adapters/whatsapp-media.ts

@@ -6,10 +6,17 @@
  * can be reused by other adapters.
  */
 
+import { createDecipheriv } from "node:crypto";
 import path from "node:path";
-import type { Readable } from "node:stream";
+import { Readable, Transform } from "node:stream";
+import type { ReadableStream as WebReadableStream } from "node:stream/web";
 import {
-  downloadMediaMessage,
+  type MediaType as BaileysMediaType,
+  DEFAULT_ORIGIN,
+  extractMessageContent,
+  getContentType,
+  getMediaKeys,
+  getUrlFromDirectPath,
   normalizeMessageContent,
   type proto,
   type WAMessage,
@@ -35,26 +42,6 @@ import type {
   MessageAttachment,
 } from "../types.js";
 
-const silentBaileysLogger: {
-  level: string;
-  child: () => typeof silentBaileysLogger;
-  trace: () => void;
-  debug: () => void;
-  info: () => void;
-  warn: () => void;
-  error: () => void;
-  fatal: () => void;
-} = {
-  level: "silent",
-  child: () => silentBaileysLogger,
-  trace: () => undefined,
-  debug: () => undefined,
-  info: () => undefined,
-  warn: () => undefined,
-  error: () => undefined,
-  fatal: () => undefined,
-};
-
 /**
  * Media info extracted from a WhatsApp message.
  * Used internally before download.
@@ -81,8 +68,8 @@ export interface MediaDownloadOptions {
    */
   onSkipped?: (info: DroppedAttachment) => void;
   /**
-   * The byte stream for a message. Default: Baileys' `downloadMediaMessage`
-   * over the socket, in stream mode. Injected only so the write legs below
+   * The byte stream for a message. Default: `fetchDecryptedMedia` — our
+   * `fetch` of the media URL, decrypted with Baileys' media keys. Injected only so the write legs below
    * can be exercised without a socket; production passes nothing.
    */
   download?: (
@@ -100,18 +87,23 @@ export interface MediaDownloadOptions {
 export interface MediaDownloadContext {
   /** Aborted on every exit but a clean end; this is what closes the socket. */
   signal: AbortSignal;
-  /** Baileys' re-upload request, under the re-upload bound instead of the open one. */
+  /**
+   * The phone re-upload, for an injected download that asks for one (production
+   * never does; see `WHATSAPP_REUPLOAD_BOUND_MS`), under the re-upload bound.
+   */
   reuploadRequest: (msg: WAMessage) => Promise<WAMessage>;
 }
 
 // The time bounds (`MediaReadBounds`, `MEDIA_READ_BOUNDS`) live in
 // `core/media.ts`, shared with the URL legs so the two readers cannot drift.
-// Baileys' `fetch` carries no timeout, so without them a stalled connection
+// The media `fetch` carries no timeout, so without them a stalled connection
 // holds the turn forever; here the idle bound is also what ends a body whose
 // connection died mid-download, because `pipe()` never tells the decrypt
 // Transform that its source errored (see `readStreamCapped`). A fired bound
-// closes the socket only through the request's `AbortSignal`, which Baileys
-// passes to `fetch` by our patch (`patches/@whiskeysockets%2Fbaileys@*.patch`).
+// closes the socket only through the request's `AbortSignal`, which is why
+// `fetchDecryptedMedia` owns the `fetch`: Baileys' `getHttpStream` takes no
+// signal, and the dependency patch that once threaded one in never reached an
+// npm-installed host (`patchedDependencies` applies to the root project only).
 
 /**
  * The bound on the phone re-upload Baileys requests when a media URL has
@@ -124,10 +116,12 @@ export interface MediaDownloadContext {
  * plain server round trip. Past it the turn reports `download_failed`, which
  * is the right answer for a phone that is offline.
  *
- * Unreachable on Baileys 7.0.0-rc14: it requests the re-upload only when
- * `error.status` is 404/410, and its own fetch error is a Boom carrying the
- * code in `output.statusCode`. `tests/whatsapp-media-socket-close.test.ts`
- * pins that fact, so a Baileys bump that makes the path live goes red there.
+ * Unreachable in production: `fetchDecryptedMedia` never requests a
+ * re-upload, which keeps the behaviour Baileys 7.0.0-rc14 had (it asked only
+ * when `error.status` was 404/410, and its own fetch error carried the code in
+ * `output.statusCode`, so never). An expired URL is `download_failed`.
+ * Making it live is a behaviour change that needs a real old quoted message
+ * to verify against; `tests/whatsapp-media-socket-close.test.ts` pins the 410.
  */
 export const WHATSAPP_REUPLOAD_BOUND_MS = 2 * 60_000;
 
@@ -173,8 +167,8 @@ function destroyWithSource(stream: Readable): void {
  * The returned promise is itself the race: whichever of end, error, the cap or
  * a bound comes first settles it, and every exit clears both timers.
  *
- * Baileys' stream is a decrypt `Transform` that the HTTP body is `pipe`d
- * into. Destroying the Transform alone only unpipes the body, and Bun's fetch
+ * The download's stream is a decrypt `Transform` that the HTTP body is
+ * `pipe`d into (`fetchDecryptedMedia`, the shape Baileys' own download had). Destroying the Transform alone only unpipes the body, and Bun's fetch
  * keeps buffering a body nobody reads (measured 2026-10-02: 591 MB RSS for a
  * 512 MB body, against 54 MB when the source is destroyed too). The source is
  * not exposed, so it is taken from the `unpipe` event every exit triggers:
@@ -188,13 +182,13 @@ function destroyWithSource(stream: Readable): void {
  * - **No early listener.** The destination of a `pipe()` holds no reference
  *   to its source (the closures in `pipe` do), so the source cannot be
  *   reached until it unpipes. That is safe only because the listener below is
- *   attached synchronously on entry, and Baileys' path from
- *   `fetched.pipe(output)` to our caller is promise resolutions only: `pipe`
+ *   attached synchronously on entry, and `fetchDecryptedMedia`'s path from
+ *   `body.pipe(decrypt)` to our caller is promise resolutions only: `pipe`
  *   starts the flow with `process.nextTick`, which runs after the microtask
  *   queue drains, so no chunk and no decrypt error can reach the Transform
  *   before this listener exists — even with the body already buffered. A
- *   macrotask inserted between Baileys' `pipe` and its `return` would open
- *   that window; `tests/whatsapp-media-capped-read.test.ts` pins the shape.
+ *   macrotask inserted between that `pipe` and its `return` would open that
+ *   window; `tests/whatsapp-media-capped-read.test.ts` pins the shape.
  * - **A dead body is silent.** When the connection resets mid-body, Bun's
  *   `Readable.fromWeb` source closes errored and emits no `'error'`, and
  *   `pipe()` forwards neither, so the Transform simply stops receiving. No
@@ -290,7 +284,7 @@ export function readStreamCapped(
 }
 
 /**
- * The bound on Baileys' open: the media fetch up to its response headers, under
+ * The bound on the download's open: the media fetch up to its response headers, under
  * the idle bound, and a phone re-upload in between, under its own bound. One
  * timer, re-armed as the phase changes; `rejected` settles when it fires.
  */
@@ -342,6 +336,119 @@ function openBound(
   };
 }
 
+/** WhatsApp media is AES-256-CBC in 16-byte blocks, followed by a 10-byte MAC. */
+const AES_BLOCK = 16;
+
+/** Thrown for a media URL that did not answer 2xx; the caller reports `download_failed`. */
+export class MediaFetchError extends Error {
+  override readonly name = "MediaFetchError";
+  constructor(readonly status: number) {
+    super(`WhatsApp media fetch failed: HTTP ${status}`);
+  }
+}
+
+/**
+ * Decrypt WhatsApp's media encryption as a stream: feed whole AES blocks to
+ * the decipher and hold back the remainder, which at the end is the 10-byte
+ * MAC and is dropped. The same arithmetic as Baileys' `downloadEncryptedContent`
+ * (no range support: we always fetch the whole file).
+ */
+function decryptMediaStream(cipherKey: Uint8Array, iv: Uint8Array): Transform {
+  const decipher = createDecipheriv("aes-256-cbc", cipherKey, iv);
+  let remainder: Buffer = Buffer.alloc(0);
+  return new Transform({
+    transform(chunk: Buffer, _encoding, callback) {
+      const data = remainder.length ? Buffer.concat([remainder, chunk]) : chunk;
+      const whole = Math.floor(data.length / AES_BLOCK) * AES_BLOCK;
+      remainder = data.subarray(whole);
+      try {
+        callback(null, decipher.update(data.subarray(0, whole)));
+      } catch (error) {
+        callback(error as Error);
+      }
+    },
+    flush(callback) {
+      try {
+        callback(null, decipher.final());
+      } catch (error) {
+        callback(error as Error);
+      }
+    },
+  });
+}
+
+/** `new URL(url).host`, or undefined for a missing or unparsable URL. */
+function hostOf(url: string | null | undefined): string | undefined {
+  if (!url) return undefined;
+  try {
+    return new URL(url).host;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * The production download: the media URL, fetched by us so the request
+ * carries `context.signal` (the only thing that closes its socket), and piped
+ * through the decrypt. Baileys supplies what it is the authority on — which
+ * content is the media, the URL from its `directPath`, the keys — and none of
+ * the I/O. Mirrors Baileys 7.0.0-rc14's `downloadMediaMessage` except that it
+ * never requests a phone re-upload (see `WHATSAPP_REUPLOAD_BOUND_MS`).
+ */
+async function fetchDecryptedMedia(
+  msg: WAMessage,
+  _sock: WASocket,
+  context: MediaDownloadContext,
+): Promise<Readable> {
+  const content = extractMessageContent(msg.message);
+  const contentType = content ? getContentType(content) : undefined;
+  const media = (content && contentType ? content[contentType] : undefined) as
+    | {
+        url?: string | null;
+        directPath?: string | null;
+        thumbnailDirectPath?: string | null;
+        mediaKey?: Uint8Array | null;
+      }
+    | undefined;
+  if (
+    !contentType ||
+    !media ||
+    typeof media !== "object" ||
+    (!("url" in media) && !("thumbnailDirectPath" in media))
+  ) {
+    throw new Error(`"${contentType}" message is not a media message`);
+  }
+  // Baileys' keying: the HKDF info string comes from the content type, and a
+  // wrong one decrypts to garbage silently (CBC does not authenticate).
+  let mediaType = contentType.replace("Message", "") as BaileysMediaType;
+  let directPath = media.directPath;
+  if ("thumbnailDirectPath" in media && !("url" in media)) {
+    directPath = media.thumbnailDirectPath;
+    mediaType = "thumbnail-link";
+  }
+  const url = directPath
+    ? getUrlFromDirectPath(directPath, hostOf(media.url))
+    : media.url;
+  if (!url) throw new Error("No media URL or directPath in the message");
+
+  const { cipherKey, iv } = await getMediaKeys(media.mediaKey, mediaType);
+  const response = await fetch(url, {
+    signal: context.signal,
+    headers: { Origin: DEFAULT_ORIGIN },
+  });
+  if (!response.ok || !response.body) {
+    // A non-2xx body is never read; `downloadBytes`' abort closes the socket.
+    response.body?.cancel().catch((error: unknown) =>
+      logger.debug("WhatsApp media non-2xx body cancel failed", {
+        error: error instanceof Error ? error.message : String(error),
+      }),
+    );
+    throw new MediaFetchError(response.status);
+  }
+  const body = Readable.fromWeb(response.body as WebReadableStream);
+  return body.pipe(decryptMediaStream(cipherKey, iv));
+}
+
 async function downloadBytes(
   msg: WAMessage,
   sock: WASocket,
@@ -352,7 +459,7 @@ async function downloadBytes(
     bounds.idleMs,
     options.reuploadBoundMs ?? WHATSAPP_REUPLOAD_BOUND_MS,
   );
-  // Destroying the stream Baileys returns, or its HTTP source, releases the
+  // Destroying the download's stream, or its HTTP source, releases the
   // reader but leaves the socket draining the body (measured on Bun 1.3.14:
   // still receiving after the destroy, closed within 1 ms of an abort). So
   // every exit aborts; after a clean end the abort is a no-op. The abort can
@@ -375,18 +482,11 @@ async function downloadBytes(
   };
   try {
     open.arm("open");
-    const opening = options.download
-      ? options.download(msg, sock, context)
-      : (downloadMediaMessage(
-          msg,
-          "stream",
-          { options: { signal: controller.signal } },
-          {
-            // biome-ignore lint/suspicious/noExplicitAny: Baileys logger type is complex
-            logger: silentBaileysLogger as any,
-            reuploadRequest: context.reuploadRequest,
-          },
-        ) as Promise<Readable>);
+    const opening = (options.download ?? fetchDecryptedMedia)(
+      msg,
+      sock,
+      context,
+    );
     // A stream that arrives after the bound fired is torn down on arrival.
     opening.then(
       (stream) => {