@blamejs/core 0.11.24 → 0.11.26

package/lib/mail-server-submission.js

@@ -428,9 +428,27 @@ function create(opts) {
       authPending:   null,
     };
 
-    var lineBuffer = "";
+    // RAW byte buffer — NOT a string. The BDAT-CHUNKING path (RFC 3030)
+    // requires lossless byte preservation when the BDAT command line +
+    // payload arrive in the same TCP segment, and DATA-body 8BITMIME
+    // payloads can contain bytes that are invalid UTF-8. Decoding the
+    // socket-bytes through a string layer replaces invalid sequences
+    // with U+FFFD and corrupts the body. Keep the raw bytes; decode to
+    // string only for the per-command parse.
+    var lineBuffer = Buffer.alloc(0);
     var bodyCollector = null;
     var inDataBody = false;
+    // RFC 3030 CHUNKING — state for the BDAT command. `bdatCollector`
+    // accumulates the message body across multiple BDAT chunks; it lives
+    // for the lifetime of the SMTP transaction (i.e., between MAIL FROM
+    // and the BDAT ... LAST that finalises). `bdatRemaining` counts down
+    // bytes still owed by the current BDAT chunk; `bdatIsLast` flags
+    // whether the current chunk is the terminator.
+    var inBdatChunk    = false;
+    var bdatRemaining  = 0;
+    var bdatIsLast     = false;
+    var bdatCollector  = null;
+    var bdatTotalBytes = 0;
 
     socket.setTimeout(idleTimeoutMs);
     socket.on("timeout", function () {
@@ -465,6 +483,57 @@ function create(opts) {
     });
 
     function _ingestBytes(state, socket, chunk) {
+      // RFC 3030 — when a BDAT chunk is in progress we consume exactly
+      // `bdatRemaining` bytes off the wire, no dot-stuffing, no end-of-
+      // data marker. Any excess bytes in the chunk after the BDAT
+      // payload completes get fed back through the command line buffer
+      // (typical when a pipelined `BDAT N LAST\r\n<payload>\r\nNOOP\r\n`
+      // arrives in a single TCP segment).
+      if (inBdatChunk) {
+        var consumeN = Math.min(chunk.length, bdatRemaining);
+        var consumed = chunk.subarray(0, consumeN);
+        try { bdatCollector.push(consumed); }
+        catch (_e) {
+          _emit("mail.server.submission.bdat_refused",
+            { connectionId: state.id, reason: "body-too-large", maxBytes: maxMessageBytes },
+            "denied");
+          _writeReply(socket, REPLY_552_SIZE_EXCEEDED,
+            "5.3.4 BDAT body exceeds maxMessageBytes (" + maxMessageBytes + " bytes)");
+          _resetTransaction(state);
+          inBdatChunk = false; bdatCollector = null; bdatRemaining = 0; bdatTotalBytes = 0;
+          return;
+        }
+        bdatRemaining -= consumeN;
+        bdatTotalBytes += consumeN;
+        if (bdatRemaining === 0) {
+          var wasLast = bdatIsLast;
+          inBdatChunk = false;
+          if (wasLast) {
+            // RFC 3030 §2.2 — ONE reply per BDAT command. When LAST,
+            // the single reply is the "message queued" finalize reply
+            // (emitted from _finalizeAcceptedBody), not the per-chunk
+            // "<N> octets received" reply. Emitting both would
+            // desynchronise the client (the second 250 would be
+            // consumed as the response to the next command).
+            // No dot-unstuff for BDAT — RFC 3030 §3 explicitly defines
+            // BDAT payloads as opaque byte streams.
+            var bdatBody = bdatCollector.result();
+            bdatCollector = null;
+            bdatTotalBytes = 0;
+            _finalizeAcceptedBody(state, socket, bdatBody, "BDAT");
+          } else {
+            // Non-final chunk — per-chunk acknowledgement only.
+            _writeReply(socket, REPLY_250_OK,
+              "2.0.0 " + bdatTotalBytes + " octets received");
+          }
+          // Any tail bytes after this BDAT chunk get re-fed as commands.
+          if (consumeN < chunk.length) {
+            var tail = chunk.subarray(consumeN);
+            _ingestBytes(state, socket, tail);
+          }
+        }
+        return;
+      }
       if (inDataBody) {
         try { bodyCollector.push(chunk); }
         catch (_e) {
@@ -491,13 +560,15 @@ function create(opts) {
         var endIdx = safeSmtp.findDotTerminator(collected);
         if (endIdx !== -1) {
           var body = collected.subarray(0, endIdx);
-          _finalizeDataBody(state, socket, body);
+          // DATA path dot-unstuffs here; BDAT path skips this step.
+          var dedotted = safeSmtp.dotUnstuff(body);
+          _finalizeAcceptedBody(state, socket, dedotted, "DATA");
           inDataBody = false; bodyCollector = null;
         }
         return;
       }
 
-      lineBuffer += chunk.toString("utf8");
+      lineBuffer = lineBuffer.length === 0 ? chunk : Buffer.concat([lineBuffer, chunk]);
       if (lineBuffer.length > maxLineBytes * 4) {
         _writeReply(socket, REPLY_500_SYNTAX,
           "5.5.6 Line too long (>" + maxLineBytes + " bytes)");
@@ -505,11 +576,29 @@ function create(opts) {
         return;
       }
       var crlf;
-      while ((crlf = lineBuffer.indexOf("\r\n")) !== -1) {
-        var line = lineBuffer.slice(0, crlf);
-        lineBuffer = lineBuffer.slice(crlf + 2);
+      var crlfNeedle = Buffer.from("\r\n", "ascii");
+      while ((crlf = lineBuffer.indexOf(crlfNeedle)) !== -1) {
+        // Decode just the per-command line to a string — keeps the
+        // wire-protocol parser working in UTF-8 while leaving the
+        // RAW lineBuffer intact for any binary payload that follows.
+        var line = lineBuffer.subarray(0, crlf).toString("utf8");
+        lineBuffer = lineBuffer.subarray(crlf + 2);
         _handleCommand(state, socket, line);
         if (inDataBody) return;
+        if (inBdatChunk) {
+          // RFC 3030 — `BDAT <N> [LAST]\r\n` is immediately followed by
+          // exactly <N> raw bytes (no dot-stuffing, no terminator). When
+          // those bytes arrived in the SAME TCP segment as the BDAT
+          // command, drain them straight from the raw byte buffer
+          // (NOT through a UTF-8 string round-trip — would corrupt
+          // 8-bit / binary payloads).
+          if (lineBuffer.length > 0) {
+            var pendingBytes = lineBuffer;
+            lineBuffer = Buffer.alloc(0);
+            _ingestBytes(state, socket, pendingBytes);
+          }
+          return;
+        }
       }
     }
 
@@ -555,6 +644,8 @@ function create(opts) {
           return _handleRcptTo(state, socket, line);
         case "DATA":
           return _handleData(state, socket);
+        case "BDAT":
+          return _handleBdat(state, socket, line);
         case "NOOP":
           return _writeReply(socket, REPLY_250_OK, "2.0.0 OK");
         case "RSET":
@@ -592,7 +683,7 @@ function create(opts) {
       state.helo  = helo;
       state.stage = "ehlo";
       if (verb === "EHLO") {
-        var caps = ["PIPELINING", "SIZE " + maxMessageBytes, "8BITMIME", "ENHANCEDSTATUSCODES"];
+        var caps = ["PIPELINING", "SIZE " + maxMessageBytes, "8BITMIME", "ENHANCEDSTATUSCODES", "CHUNKING"];
         // STARTTLS advertised only on explicit-STARTTLS port (587),
         // not on implicit-TLS (465 already wrapped). RFC 8314 §3.3.
         if (!state.tls && !implicitTls) caps.unshift("STARTTLS");
@@ -627,7 +718,12 @@ function create(opts) {
       // body collector AND strip the plain-socket "data" listener
       // before wrapping in TLSSocket so bytes the peer pipelined
       // pre-handshake cannot reach the post-TLS state machine.
-      lineBuffer = ""; bodyCollector = null; inDataBody = false;
+      lineBuffer = Buffer.alloc(0); bodyCollector = null; inDataBody = false;
+      // BDAT-side state cleared on STARTTLS upgrade too — same threat
+      // model as CVE-2021-38371 (Exim) / CVE-2021-33515 (Dovecot):
+      // pre-handshake bytes the peer pipelined MUST NOT reach the
+      // post-TLS state machine via the BDAT collector either.
+      inBdatChunk = false; bdatRemaining = 0; bdatCollector = null; bdatTotalBytes = 0;
       mailServerTls.upgradeSocket({
         plainSocket:   socket,
         secureContext: opts.tlsContext,
@@ -1033,8 +1129,7 @@ function create(opts) {
       });
     }
 
-    function _finalizeDataBody(state, socket, body) {
-      var dedotted = safeSmtp.dotUnstuff(body);
+    function _finalizeAcceptedBody(state, socket, dedotted, source) {
 
       // Outbound DKIM-required gate. Scan the header block for a
       // `DKIM-Signature:` line; under `self` mode also require at
@@ -1108,16 +1203,109 @@ function create(opts) {
       }
       _emit("mail.server.submission.data_accepted",
         { connectionId: state.id, mailFrom: state.mailFrom,
-          rcptCount: state.rcpts.length, sizeBytes: dedotted.length });
+          rcptCount: state.rcpts.length, sizeBytes: dedotted.length, source: source || "DATA" });
       _writeReply(socket, REPLY_250_OK, "2.6.0 Message queued (audit-only)");
       _resetTransaction(state);
     }
 
+    // RFC 3030 §2 — BDAT <chunk-size> [LAST]. Reads exactly chunk-size
+    // bytes off the wire (no dot-stuffing, no end-of-data marker). The
+    // size is a non-negative integer; LAST keyword (case-insensitive)
+    // terminates the message body. Mixing DATA + BDAT within the same
+    // transaction is forbidden — the server returns 503 once the first
+    // BDAT lands and forces the client to RSET.
+    function _handleBdat(state, socket, line) {
+      if (state.stage !== "rcpt" && state.stage !== "bdat") {
+        _writeReply(socket, REPLY_503_BAD_SEQUENCE, "5.5.1 BDAT requires MAIL FROM + RCPT TO");
+        return;
+      }
+      if (state.rcpts.length === 0) {
+        _writeReply(socket, REPLY_503_BAD_SEQUENCE, "5.5.1 No valid recipients");
+        return;
+      }
+      // Pipelining race — same gate as DATA.
+      if ((state.rcptsPending || 0) > 0) {
+        _emit("mail.server.submission.pipelining_bdat_race", {
+          connectionId: state.id, rcptsPending: state.rcptsPending,
+          rcptsCommitted: state.rcpts.length,
+        }, "denied");
+        _writeReply(socket, REPLY_451_LOCAL_ERROR,
+          "4.5.0 RCPT TO verdicts pending; reissue BDAT after recipient replies");
+        return;
+      }
+      // Parse `BDAT <size>[ LAST]`.
+      var parts = line.split(/\s+/);
+      if (parts.length < 2 || parts.length > 3) {
+        _writeReply(socket, REPLY_501_BAD_ARGS, "5.5.4 BDAT requires <chunk-size> [LAST]");
+        return;
+      }
+      var sizeStr = parts[1];
+      var sizeN = parseInt(sizeStr, 10);
+      if (!/^\d+$/.test(sizeStr) || !isFinite(sizeN) || sizeN < 0) {
+        _writeReply(socket, REPLY_501_BAD_ARGS, "5.5.4 BDAT chunk-size must be a non-negative integer");
+        return;
+      }
+      var isLast = parts.length === 3 && parts[2].toUpperCase() === "LAST";
+      if (parts.length === 3 && !isLast) {
+        _writeReply(socket, REPLY_501_BAD_ARGS, "5.5.4 BDAT third arg must be 'LAST' (RFC 3030 §2)");
+        return;
+      }
+      // Cumulative-size cap. The collector is bounded too, but checking
+      // up-front lets us refuse the chunk before reading bytes off the
+      // socket — important when sizeN >> maxMessageBytes.
+      if (bdatTotalBytes + sizeN > maxMessageBytes) {
+        _emit("mail.server.submission.bdat_refused",
+          { connectionId: state.id, reason: "body-too-large",
+            requestedTotal: bdatTotalBytes + sizeN, maxBytes: maxMessageBytes }, "denied");
+        _writeReply(socket, REPLY_552_SIZE_EXCEEDED,
+          "5.3.4 BDAT cumulative size " + (bdatTotalBytes + sizeN) +
+          " exceeds maxMessageBytes (" + maxMessageBytes + ")");
+        _resetTransaction(state);
+        bdatCollector = null; bdatTotalBytes = 0;
+        return;
+      }
+      if (!bdatCollector) {
+        bdatCollector = safeBuffer.boundedChunkCollector({
+          maxBytes:    maxMessageBytes,
+          errorClass:  MailServerSubmissionError,
+          sizeCode:    "mail-server-submission/body-too-large",
+          sizeMessage: "BDAT body exceeded maxMessageBytes (" + maxMessageBytes + ")",
+        });
+      }
+      state.stage   = "bdat";
+      bdatRemaining = sizeN;
+      bdatIsLast    = isLast;
+      // size=0 + LAST is a valid sequence — finalises the message
+      // body (the LAST chunk may carry zero bytes when the prior chunk
+      // was the final payload). RFC 3030 §2.2 — ONE reply per command:
+      // emit the "0 octets" ack for size=0 NOT-LAST, but defer to
+      // _finalizeAcceptedBody for size=0 LAST.
+      if (sizeN === 0) {
+        if (isLast) {
+          var emptyBody = bdatCollector ? bdatCollector.result() : Buffer.alloc(0);
+          bdatCollector = null; bdatTotalBytes = 0;
+          _finalizeAcceptedBody(state, socket, emptyBody, "BDAT");
+        } else {
+          _writeReply(socket, REPLY_250_OK, "2.0.0 0 octets received");
+        }
+        return;
+      }
+      inBdatChunk = true;
+    }
+
     function _resetTransaction(state) {
       state.mailFrom     = null;
       state.rcpts        = [];
       state.rcptsPending = 0;
       state.stage        = "ehlo";
+      // BDAT-side state lives at the connection level, not on `state`.
+      // Reset it here so a RSET / failed BDAT can't leak collected
+      // bytes into the next transaction.
+      inBdatChunk    = false;
+      bdatRemaining  = 0;
+      bdatIsLast     = false;
+      bdatCollector  = null;
+      bdatTotalBytes = 0;
     }
   }
 

package/lib/mail-store-fts.js

@@ -0,0 +1,394 @@
+"use strict";
+/**
+ * @module    b.mailStore.fts
+ * @nav       Mail
+ * @title     Mail-store FTS (sealed-token full-text index)
+ * @order     250
+ * @slug      mail-store-fts
+ *
+ * @intro
+ *   Sealed-token full-text search index for `b.mailStore`. At
+ *   `appendMessage` time the row's plaintext subject + addresses + body
+ *   are tokenized, each token is hashed with the per-deployment vault
+ *   salt (the same scheme `b.cryptoField` uses for derived-hash mirrors
+ *   on sealed columns), and the resulting space-separated token-hash
+ *   string is inserted into a SQLite FTS5 virtual table. Search runs
+ *   the same tokenize → hash transform on the operator's query terms
+ *   and issues `MATCH` against the FTS5 table — never against
+ *   plaintext.
+ *
+ *   The index is unrecoverable without the vault salt. A database
+ *   dump leaks zero readable text — the FTS5 rows are byte-for-byte
+ *   indistinguishable from random hashes. Per-tenant separation rides
+ *   on the cryptoField namespace prefix (`bj-<table>-<field>:`), so
+ *   tokens from one tenant's row can never collide with another's
+ *   under the same vault key.
+ *
+ *   Limitations of sealed-token FTS — operator-facing constraints:
+ *
+ *     - Exact-token match only. No SQLite FTS5 stemmer, no porter,
+ *       no Unicode-fold-then-stem, no NEAR with offsets. The token
+ *       boundary IS the search granularity. Operators that need
+ *       linguistic search at the cost of plaintext-at-rest opt in to
+ *       a separate plaintext-FTS layer on top — not part of this
+ *       primitive.
+ *     - No prefix wildcard (`MATCH 'kub*'`). Token hashes don't
+ *       preserve substring relationships. The cost of partial-match
+ *       search is sealed-at-rest; operators get either-or.
+ *     - Stopword filter is conservative (a / the / of / to / in /
+ *       for / on / and / or / is / are / be / by). Stopwords land
+ *       in the unsealed plaintext but never reach the FTS row.
+ *     - Token length capped at 2..64 unicode codepoints after
+ *       NFC normalisation. Tokens outside the band are dropped (too
+ *       short = high-collision noise; too long = file-bomb shape).
+ *
+ *   Posture cascade. The primitive is on by default for every
+ *   posture (`hipaa` / `pci-dss` / `gdpr` / `soc2`) — the token
+ *   index uses the same vault key already protecting sealed-row
+ *   storage, so adding the FTS index doesn't widen the cryptographic
+ *   trust boundary. A future opt-in plaintext-FTS overlay would be
+ *   gated by a relaxed posture; this module ships sealed-only.
+ *
+ * @card
+ *   Tokenize → vault-salted hash → FTS5 MATCH. The DB dump leaks
+ *   nothing readable; search works against ciphertext.
+ */
+
+var bCrypto      = require("./crypto");
+var vault        = require("./vault");
+var C            = require("./constants");
+
+// Stopwords are dropped before hashing — they'd dominate every row's
+// token set without adding query selectivity. Kept conservative to
+// stay locale-neutral for v1.
+var STOPWORDS = Object.create(null);
+(
+  "a an the of to in for on and or but is are was were be been by with " +
+  "as at from this that it its their our your his her him us we i you " +
+  "do does did not no yes if so up down out over under than then them"
+).split(" ").forEach(function (w) { STOPWORDS[w] = true; });
+
+// Per-token bounds. NFC-normalised codepoint count, not byte length —
+// tokens carrying multi-byte UTF-8 are not penalised relative to ASCII.
+var MIN_TOKEN_LEN = 2;
+var MAX_TOKEN_LEN = 64;
+
+// Per-row token-set cap. A single 50 MiB message can produce
+// millions of tokens; FTS5 row insert + index update must stay
+// bounded. The cap is applied AFTER stopword + length filter so the
+// surviving tokens are the highest-signal subset.
+var MAX_TOKENS_PER_FIELD = 8192;                                                                       // allow:raw-byte-literal — token-count cap, not bytes
+
+// Per-field FTS column names. Kept symmetric with the messages table
+// columns so callers can reason about which FTS column corresponds
+// to which plaintext source field.
+var FTS_FIELDS = {
+  subject:  "subject_toks",
+  from:     "addr_toks",
+  to:       "addr_toks",
+  body:     "body_toks",
+};
+
+// Token splitter — Unicode-aware. Splits on every non-letter,
+// non-digit code-point (Unicode category L* or N*). Apostrophes inside
+// a word survive (`don't` → `don't`); leading/trailing punctuation is
+// stripped. Email addresses are split on `@` + `.` so both the local
+// part and each domain label produce independent tokens — operators
+// searching for `example.com` find rows whose from/to header carries
+// `alice@example.com` AND rows that mention `example` or `com` in
+// body prose. Stopwords prune the noisy ones.
+//
+// Refuses input larger than MAX_INPUT_BYTES to bound tokenizer work
+// — protects against DoS-shaped messages whose body is 50 MiB of a
+// single token boundary.
+var MAX_INPUT_BYTES = C.BYTES.mib(8);                         // 8 MiB
+
+/**
+ * @primitive b.mailStore.fts.tokenize
+ * @signature b.mailStore.fts.tokenize(text)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Split `text` into a deduplicated, lowercased, NFC-normalised token
+ * array. Drops stopwords + tokens outside the 2..64-codepoint band.
+ * Splits on every non-letter / non-digit codepoint, including the
+ * `@` + `.` boundaries of email addresses so local-part + domain
+ * labels become independent tokens.
+ *
+ * @example
+ *   b.mailStore.fts.tokenize("Hello world from alice@example.com");
+ *   // → ["hello", "world", "alice", "example", "com"]
+ */
+function tokenize(text) {
+  if (typeof text !== "string") return [];
+  if (text.length === 0) return [];
+  if (Buffer.byteLength(text, "utf8") > MAX_INPUT_BYTES) {
+    // Truncate at MAX_INPUT_BYTES. Tokenization on the prefix is
+    // already representative for the body's content fingerprint;
+    // refusing outright would weaken indexing on legitimately large
+    // messages.
+    text = text.slice(0, MAX_INPUT_BYTES);
+  }
+  // NFC normalise so visually-identical tokens hash to the same value
+  // regardless of the source's encoding form.
+  var nfc = text.normalize("NFC").toLowerCase();
+  // Split on any run of characters that is NOT a letter, digit, or
+  // intra-word apostrophe. `\p{L}` + `\p{N}` need the `u` flag.
+  var rawTokens = nfc.split(/[^\p{L}\p{N}']+/u);
+  var seen = Object.create(null);
+  var out = [];
+  for (var i = 0; i < rawTokens.length && out.length < MAX_TOKENS_PER_FIELD; i++) {
+    var t = rawTokens[i];
+    if (!t) continue;
+    // Drop leading/trailing apostrophes that survived the split.
+    t = t.replace(/^[']+/, "").replace(/[']+$/, "");
+    if (!t) continue;
+    // Count CODEPOINTS, not UTF-16 units.
+    var len = Array.from(t).length;
+    if (len < MIN_TOKEN_LEN || len > MAX_TOKEN_LEN) continue;
+    if (STOPWORDS[t]) continue;
+    if (seen[t]) continue;
+    seen[t] = true;
+    out.push(t);
+  }
+  return out;
+}
+
+// Hash one token using the same scheme cryptoField uses for derived-
+// hash mirrors: `sha3Hash(vaultSalt + namespace + token)`. The
+// namespace is per-table, per-field, per-purpose ("fts") so that
+// rotating an operator's vault salt invalidates every FTS row in
+// the same step as every sealed column. Returns a 16-char hex prefix
+// — full 64-char SHA3 is overkill for FTS hash space, and shorter
+// tokens compress the FTS5 row 4x without observable collision risk
+// at corpus sizes the framework targets (≤ 10^9 unique tokens, where
+// 64-bit collision space leaves the birthday bound > 10^9).
+/**
+ * @primitive b.mailStore.fts.hashToken
+ * @signature b.mailStore.fts.hashToken(table, field, token)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Vault-salted hash of one token under the (table, field) namespace.
+ * The same scheme `b.cryptoField.computeDerived` uses for derived-
+ * hash mirrors on sealed columns — rotating the vault salt
+ * invalidates every FTS hash in step with every sealed-column hash.
+ * Returns a 16-char hex prefix.
+ *
+ * @example
+ *   var h = b.mailStore.fts.hashToken("mail_messages", "body", "kubernetes");
+ *   /^[0-9a-f]{16}$/.test(h);   // → true
+ */
+function hashToken(table, field, token) {
+  if (typeof token !== "string" || token.length === 0) return "";
+  // Mirrors cryptoField's internal `namespaceFor()` scheme — the FTS
+  // fields are pseudo-fields (no sealed-column registration), so the
+  // canonical fallback path is always the right answer here.
+  var ns = "bj-" + table + "-" + field + ":fts:";
+  var salt = vault.getDerivedHashSalt();
+  var saltHex = (salt && typeof salt.toString === "function") ? salt.toString("hex") : "";
+  return bCrypto.sha3Hash(saltHex + ns + token).slice(0, 16);                                          // allow:raw-byte-literal — 16-char hex prefix length, not bytes
+}
+
+// Hash a token array → space-separated string suitable for FTS5
+// row insertion. The output is what gets MATCH'd at query time.
+/**
+ * @primitive b.mailStore.fts.hashTokens
+ * @signature b.mailStore.fts.hashTokens(table, field, tokens)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Hash an array of tokens → space-separated hash string suitable for
+ * direct insertion into an FTS5 column. Empty + duplicate token-
+ * hashes drop on the way out.
+ *
+ * @example
+ *   b.mailStore.fts.hashTokens("t", "subject", ["hello", "world"]);
+ *   // → "<16hex> <16hex>"
+ */
+function hashTokens(table, field, tokens) {
+  if (!Array.isArray(tokens) || tokens.length === 0) return "";
+  var seen = Object.create(null);
+  var out = [];
+  for (var i = 0; i < tokens.length; i++) {
+    var h = hashToken(table, field, tokens[i]);
+    if (!h || seen[h]) continue;
+    seen[h] = true;
+    out.push(h);
+  }
+  return out.join(" ");
+}
+
+// Tokenize + hash + join in one step (the common path for both
+// append-side index updates and search-side query rewriting).
+/**
+ * @primitive b.mailStore.fts.hashText
+ * @signature b.mailStore.fts.hashText(table, field, text)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Tokenize + hash + join in one step. Convenience wrapper —
+ * equivalent to `hashTokens(table, field, tokenize(text))`.
+ *
+ * @example
+ *   b.mailStore.fts.hashText("mail_messages", "body", "kubernetes deploy");
+ *   // → "<16hex> <16hex>"
+ */
+function hashText(table, field, text) {
+  return hashTokens(table, field, tokenize(text));
+}
+
+// Build the FTS row body for one message. Subject + body tokens get
+// their own FTS columns; from + to addresses share `addr_toks` so a
+// search for an address hits regardless of which side it's on. The
+// `addr_toks` namespace is a single pseudo-field "addr" so the index
+// + query sides hash identically regardless of which header carried
+// the token — `{from: "alice@x"}` and `{to: "alice@x"}` BOTH hit a
+// row that mentions alice@x in EITHER header.
+/**
+ * @primitive b.mailStore.fts.rowFromMessage
+ * @signature b.mailStore.fts.rowFromMessage(table, msg)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Build the FTS5 row payload `{ objectid, subject_toks, addr_toks,
+ * body_toks }` from a `{ objectid, subject, from, to, body }`
+ * plaintext message. `from` + `to` share `addr_toks`.
+ *
+ * @example
+ *   b.mailStore.fts.rowFromMessage("t", { objectid:"o1", subject:"Hi", from:"a@x", to:"b@x", body:"hello" });
+ *   // → { objectid:"o1", subject_toks:"<hash>", addr_toks:"<hash> <hash>", body_toks:"<hash>" }
+ */
+function rowFromMessage(table, msg) {
+  var addrTokens = tokenize(msg.from || "").concat(tokenize(msg.to || ""));
+  return {
+    objectid:     msg.objectid,
+    subject_toks: hashText(table, "subject", msg.subject || ""),
+    addr_toks:    hashTokens(table, "addr", addrTokens),
+    body_toks:    hashText(table, "body", msg.body || ""),
+  };
+}
+
+// Map a query-side filter key onto the (FTS5 column, namespace pseudo-
+// field) pair the indexer used. Keeps the index + query in lock-step
+// so future column additions only touch this table.
+//
+//   filter key   →  FTS5 column    +  namespace field
+//   subject      →  subject_toks   +  "subject"
+//   body         →  body_toks      +  "body"
+//   from / to    →  addr_toks      +  "addr"
+//
+// For a broad cross-column `text` query the caller iterates this
+// mapping and OR's the per-column MATCH clauses.
+var QUERY_KEY_MAP = {
+  subject: { column: "subject_toks", field: "subject" },
+  body:    { column: "body_toks",    field: "body" },
+  from:    { column: "addr_toks",    field: "addr" },
+  to:      { column: "addr_toks",    field: "addr" },
+};
+
+/**
+ * @primitive b.mailStore.fts.columnAndFieldFor
+ * @signature b.mailStore.fts.columnAndFieldFor(filterKey)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Map a search filter key (`subject` / `body` / `from` / `to`) to
+ * the FTS5 column it indexes into PLUS the namespace pseudo-field
+ * the indexer uses when hashing tokens. Used by the search path so
+ * the query-side hash transform matches the index-side one byte-
+ * for-byte.
+ *
+ * @example
+ *   b.mailStore.fts.columnAndFieldFor("from");
+ *   // → { column: "addr_toks", field: "addr" }
+ */
+function columnAndFieldFor(key) {
+  return QUERY_KEY_MAP[key] || null;
+}
+
+// Rewrite an operator query term into a FTS5 MATCH expression. The
+// term is tokenized + hashed exactly like an index value, then the
+// hashes are AND'd together so multi-word queries require every
+// token to appear in the row. Returns null when no tokens survive
+// the filter (caller should skip the FTS join in that case).
+/**
+ * @primitive b.mailStore.fts.buildMatchExpression
+ * @signature b.mailStore.fts.buildMatchExpression(table, field, term)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Tokenize + hash an operator's query `term` and produce the FTS5
+ * MATCH expression that selects rows containing every surviving
+ * token. Returns `null` when no tokens survive the tokenize +
+ * stopword filter (caller skips the FTS join in that case).
+ *
+ * @example
+ *   var expr = b.mailStore.fts.buildMatchExpression("t", "body", "kubernetes deploy");
+ *   // → "<16hex> AND <16hex>"
+ */
+function buildMatchExpression(table, field, term) {
+  var tokens = tokenize(term);
+  if (tokens.length === 0) return null;
+  var hashes = [];
+  var seen = Object.create(null);
+  for (var i = 0; i < tokens.length; i++) {
+    var h = hashToken(table, field, tokens[i]);
+    if (!h || seen[h]) continue;
+    seen[h] = true;
+    hashes.push(h);
+  }
+  if (hashes.length === 0) return null;
+  // FTS5 default operator is AND; explicit for readability.
+  return hashes.join(" AND ");
+}
+
+// SQL builder — creates the FTS5 virtual table. Caller supplies the
+// quoted parent table identifier; this module owns the FTS table
+// name and column layout.
+/**
+ * @primitive b.mailStore.fts.createSql
+ * @signature b.mailStore.fts.createSql(qFtsTable)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Returns the `CREATE VIRTUAL TABLE IF NOT EXISTS` SQL for the
+ * sealed-token FTS5 table. The caller passes the quoted table
+ * identifier (e.g. `"blamejs_mail_messages_fts"`).
+ *
+ * @example
+ *   db.prepare(b.mailStore.fts.createSql('"mail_fts"')).run();
+ */
+function createSql(qFtsTable) {
+  return "CREATE VIRTUAL TABLE IF NOT EXISTS " + qFtsTable + " USING fts5(" +
+    "objectid UNINDEXED, " +
+    "subject_toks, " +
+    "addr_toks, " +
+    "body_toks, " +
+    "tokenize = 'unicode61 remove_diacritics 2'" +
+  ")";
+}
+
+module.exports = {
+  // SQL primitives
+  createSql:           createSql,
+
+  // Index-side
+  tokenize:            tokenize,
+  hashToken:           hashToken,
+  hashTokens:          hashTokens,
+  hashText:            hashText,
+  rowFromMessage:      rowFromMessage,
+
+  // Query-side
+  buildMatchExpression: buildMatchExpression,
+  columnAndFieldFor:   columnAndFieldFor,
+  QUERY_KEY_MAP:       QUERY_KEY_MAP,
+
+  // Constants surfaced for tests + adjacent modules.
+  STOPWORDS:           STOPWORDS,
+  MIN_TOKEN_LEN:       MIN_TOKEN_LEN,
+  MAX_TOKEN_LEN:       MAX_TOKEN_LEN,
+  MAX_TOKENS_PER_FIELD: MAX_TOKENS_PER_FIELD,
+  FTS_FIELDS:          FTS_FIELDS,
+};