@blamejs/core 0.11.24 → 0.11.25

package/lib/mail-store-fts.js

@@ -0,0 +1,394 @@
+"use strict";
+/**
+ * @module    b.mailStore.fts
+ * @nav       Mail
+ * @title     Mail-store FTS (sealed-token full-text index)
+ * @order     250
+ * @slug      mail-store-fts
+ *
+ * @intro
+ *   Sealed-token full-text search index for `b.mailStore`. At
+ *   `appendMessage` time the row's plaintext subject + addresses + body
+ *   are tokenized, each token is hashed with the per-deployment vault
+ *   salt (the same scheme `b.cryptoField` uses for derived-hash mirrors
+ *   on sealed columns), and the resulting space-separated token-hash
+ *   string is inserted into a SQLite FTS5 virtual table. Search runs
+ *   the same tokenize → hash transform on the operator's query terms
+ *   and issues `MATCH` against the FTS5 table — never against
+ *   plaintext.
+ *
+ *   The index is unrecoverable without the vault salt. A database
+ *   dump leaks zero readable text — the FTS5 rows are byte-for-byte
+ *   indistinguishable from random hashes. Per-tenant separation rides
+ *   on the cryptoField namespace prefix (`bj-<table>-<field>:`), so
+ *   tokens from one tenant's row can never collide with another's
+ *   under the same vault key.
+ *
+ *   Limitations of sealed-token FTS — operator-facing constraints:
+ *
+ *     - Exact-token match only. No SQLite FTS5 stemmer, no porter,
+ *       no Unicode-fold-then-stem, no NEAR with offsets. The token
+ *       boundary IS the search granularity. Operators that need
+ *       linguistic search at the cost of plaintext-at-rest opt in to
+ *       a separate plaintext-FTS layer on top — not part of this
+ *       primitive.
+ *     - No prefix wildcard (`MATCH 'kub*'`). Token hashes don't
+ *       preserve substring relationships. The cost of partial-match
+ *       search is sealed-at-rest; operators get either-or.
+ *     - Stopword filter is conservative (a / the / of / to / in /
+ *       for / on / and / or / is / are / be / by). Stopwords land
+ *       in the unsealed plaintext but never reach the FTS row.
+ *     - Token length capped at 2..64 unicode codepoints after
+ *       NFC normalisation. Tokens outside the band are dropped (too
+ *       short = high-collision noise; too long = file-bomb shape).
+ *
+ *   Posture cascade. The primitive is on by default for every
+ *   posture (`hipaa` / `pci-dss` / `gdpr` / `soc2`) — the token
+ *   index uses the same vault key already protecting sealed-row
+ *   storage, so adding the FTS index doesn't widen the cryptographic
+ *   trust boundary. A future opt-in plaintext-FTS overlay would be
+ *   gated by a relaxed posture; this module ships sealed-only.
+ *
+ * @card
+ *   Tokenize → vault-salted hash → FTS5 MATCH. The DB dump leaks
+ *   nothing readable; search works against ciphertext.
+ */
+
+var bCrypto      = require("./crypto");
+var vault        = require("./vault");
+var C            = require("./constants");
+
+// Stopwords are dropped before hashing — they'd dominate every row's
+// token set without adding query selectivity. Kept conservative to
+// stay locale-neutral for v1.
+var STOPWORDS = Object.create(null);
+(
+  "a an the of to in for on and or but is are was were be been by with " +
+  "as at from this that it its their our your his her him us we i you " +
+  "do does did not no yes if so up down out over under than then them"
+).split(" ").forEach(function (w) { STOPWORDS[w] = true; });
+
+// Per-token bounds. NFC-normalised codepoint count, not byte length —
+// tokens carrying multi-byte UTF-8 are not penalised relative to ASCII.
+var MIN_TOKEN_LEN = 2;
+var MAX_TOKEN_LEN = 64;
+
+// Per-row token-set cap. A single 50 MiB message can produce
+// millions of tokens; FTS5 row insert + index update must stay
+// bounded. The cap is applied AFTER stopword + length filter so the
+// surviving tokens are the highest-signal subset.
+var MAX_TOKENS_PER_FIELD = 8192;                                                                       // allow:raw-byte-literal — token-count cap, not bytes
+
+// Per-field FTS column names. Kept symmetric with the messages table
+// columns so callers can reason about which FTS column corresponds
+// to which plaintext source field.
+var FTS_FIELDS = {
+  subject:  "subject_toks",
+  from:     "addr_toks",
+  to:       "addr_toks",
+  body:     "body_toks",
+};
+
+// Token splitter — Unicode-aware. Splits on every non-letter,
+// non-digit code-point (Unicode category L* or N*). Apostrophes inside
+// a word survive (`don't` → `don't`); leading/trailing punctuation is
+// stripped. Email addresses are split on `@` + `.` so both the local
+// part and each domain label produce independent tokens — operators
+// searching for `example.com` find rows whose from/to header carries
+// `alice@example.com` AND rows that mention `example` or `com` in
+// body prose. Stopwords prune the noisy ones.
+//
+// Refuses input larger than MAX_INPUT_BYTES to bound tokenizer work
+// — protects against DoS-shaped messages whose body is 50 MiB of a
+// single token boundary.
+var MAX_INPUT_BYTES = C.BYTES.mib(8);                         // 8 MiB
+
+/**
+ * @primitive b.mailStore.fts.tokenize
+ * @signature b.mailStore.fts.tokenize(text)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Split `text` into a deduplicated, lowercased, NFC-normalised token
+ * array. Drops stopwords + tokens outside the 2..64-codepoint band.
+ * Splits on every non-letter / non-digit codepoint, including the
+ * `@` + `.` boundaries of email addresses so local-part + domain
+ * labels become independent tokens.
+ *
+ * @example
+ *   b.mailStore.fts.tokenize("Hello world from alice@example.com");
+ *   // → ["hello", "world", "alice", "example", "com"]
+ */
+function tokenize(text) {
+  if (typeof text !== "string") return [];
+  if (text.length === 0) return [];
+  if (Buffer.byteLength(text, "utf8") > MAX_INPUT_BYTES) {
+    // Truncate at MAX_INPUT_BYTES. Tokenization on the prefix is
+    // already representative for the body's content fingerprint;
+    // refusing outright would weaken indexing on legitimately large
+    // messages.
+    text = text.slice(0, MAX_INPUT_BYTES);
+  }
+  // NFC normalise so visually-identical tokens hash to the same value
+  // regardless of the source's encoding form.
+  var nfc = text.normalize("NFC").toLowerCase();
+  // Split on any run of characters that is NOT a letter, digit, or
+  // intra-word apostrophe. `\p{L}` + `\p{N}` need the `u` flag.
+  var rawTokens = nfc.split(/[^\p{L}\p{N}']+/u);
+  var seen = Object.create(null);
+  var out = [];
+  for (var i = 0; i < rawTokens.length && out.length < MAX_TOKENS_PER_FIELD; i++) {
+    var t = rawTokens[i];
+    if (!t) continue;
+    // Drop leading/trailing apostrophes that survived the split.
+    t = t.replace(/^[']+/, "").replace(/[']+$/, "");
+    if (!t) continue;
+    // Count CODEPOINTS, not UTF-16 units.
+    var len = Array.from(t).length;
+    if (len < MIN_TOKEN_LEN || len > MAX_TOKEN_LEN) continue;
+    if (STOPWORDS[t]) continue;
+    if (seen[t]) continue;
+    seen[t] = true;
+    out.push(t);
+  }
+  return out;
+}
+
+// Hash one token using the same scheme cryptoField uses for derived-
+// hash mirrors: `sha3Hash(vaultSalt + namespace + token)`. The
+// namespace is per-table, per-field, per-purpose ("fts") so that
+// rotating an operator's vault salt invalidates every FTS row in
+// the same step as every sealed column. Returns a 16-char hex prefix
+// — full 64-char SHA3 is overkill for FTS hash space, and shorter
+// tokens compress the FTS5 row 4x without observable collision risk
+// at corpus sizes the framework targets (≤ 10^9 unique tokens, where
+// 64-bit collision space leaves the birthday bound > 10^9).
+/**
+ * @primitive b.mailStore.fts.hashToken
+ * @signature b.mailStore.fts.hashToken(table, field, token)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Vault-salted hash of one token under the (table, field) namespace.
+ * The same scheme `b.cryptoField.computeDerived` uses for derived-
+ * hash mirrors on sealed columns — rotating the vault salt
+ * invalidates every FTS hash in step with every sealed-column hash.
+ * Returns a 16-char hex prefix.
+ *
+ * @example
+ *   var h = b.mailStore.fts.hashToken("mail_messages", "body", "kubernetes");
+ *   /^[0-9a-f]{16}$/.test(h);   // → true
+ */
+function hashToken(table, field, token) {
+  if (typeof token !== "string" || token.length === 0) return "";
+  // Mirrors cryptoField's internal `namespaceFor()` scheme — the FTS
+  // fields are pseudo-fields (no sealed-column registration), so the
+  // canonical fallback path is always the right answer here.
+  var ns = "bj-" + table + "-" + field + ":fts:";
+  var salt = vault.getDerivedHashSalt();
+  var saltHex = (salt && typeof salt.toString === "function") ? salt.toString("hex") : "";
+  return bCrypto.sha3Hash(saltHex + ns + token).slice(0, 16);                                          // allow:raw-byte-literal — 16-char hex prefix length, not bytes
+}
+
+// Hash a token array → space-separated string suitable for FTS5
+// row insertion. The output is what gets MATCH'd at query time.
+/**
+ * @primitive b.mailStore.fts.hashTokens
+ * @signature b.mailStore.fts.hashTokens(table, field, tokens)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Hash an array of tokens → space-separated hash string suitable for
+ * direct insertion into an FTS5 column. Empty + duplicate token-
+ * hashes drop on the way out.
+ *
+ * @example
+ *   b.mailStore.fts.hashTokens("t", "subject", ["hello", "world"]);
+ *   // → "<16hex> <16hex>"
+ */
+function hashTokens(table, field, tokens) {
+  if (!Array.isArray(tokens) || tokens.length === 0) return "";
+  var seen = Object.create(null);
+  var out = [];
+  for (var i = 0; i < tokens.length; i++) {
+    var h = hashToken(table, field, tokens[i]);
+    if (!h || seen[h]) continue;
+    seen[h] = true;
+    out.push(h);
+  }
+  return out.join(" ");
+}
+
+// Tokenize + hash + join in one step (the common path for both
+// append-side index updates and search-side query rewriting).
+/**
+ * @primitive b.mailStore.fts.hashText
+ * @signature b.mailStore.fts.hashText(table, field, text)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Tokenize + hash + join in one step. Convenience wrapper —
+ * equivalent to `hashTokens(table, field, tokenize(text))`.
+ *
+ * @example
+ *   b.mailStore.fts.hashText("mail_messages", "body", "kubernetes deploy");
+ *   // → "<16hex> <16hex>"
+ */
+function hashText(table, field, text) {
+  return hashTokens(table, field, tokenize(text));
+}
+
+// Build the FTS row body for one message. Subject + body tokens get
+// their own FTS columns; from + to addresses share `addr_toks` so a
+// search for an address hits regardless of which side it's on. The
+// `addr_toks` namespace is a single pseudo-field "addr" so the index
+// + query sides hash identically regardless of which header carried
+// the token — `{from: "alice@x"}` and `{to: "alice@x"}` BOTH hit a
+// row that mentions alice@x in EITHER header.
+/**
+ * @primitive b.mailStore.fts.rowFromMessage
+ * @signature b.mailStore.fts.rowFromMessage(table, msg)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Build the FTS5 row payload `{ objectid, subject_toks, addr_toks,
+ * body_toks }` from a `{ objectid, subject, from, to, body }`
+ * plaintext message. `from` + `to` share `addr_toks`.
+ *
+ * @example
+ *   b.mailStore.fts.rowFromMessage("t", { objectid:"o1", subject:"Hi", from:"a@x", to:"b@x", body:"hello" });
+ *   // → { objectid:"o1", subject_toks:"<hash>", addr_toks:"<hash> <hash>", body_toks:"<hash>" }
+ */
+function rowFromMessage(table, msg) {
+  var addrTokens = tokenize(msg.from || "").concat(tokenize(msg.to || ""));
+  return {
+    objectid:     msg.objectid,
+    subject_toks: hashText(table, "subject", msg.subject || ""),
+    addr_toks:    hashTokens(table, "addr", addrTokens),
+    body_toks:    hashText(table, "body", msg.body || ""),
+  };
+}
+
+// Map a query-side filter key onto the (FTS5 column, namespace pseudo-
+// field) pair the indexer used. Keeps the index + query in lock-step
+// so future column additions only touch this table.
+//
+//   filter key   →  FTS5 column    +  namespace field
+//   subject      →  subject_toks   +  "subject"
+//   body         →  body_toks      +  "body"
+//   from / to    →  addr_toks      +  "addr"
+//
+// For a broad cross-column `text` query the caller iterates this
+// mapping and OR's the per-column MATCH clauses.
+var QUERY_KEY_MAP = {
+  subject: { column: "subject_toks", field: "subject" },
+  body:    { column: "body_toks",    field: "body" },
+  from:    { column: "addr_toks",    field: "addr" },
+  to:      { column: "addr_toks",    field: "addr" },
+};
+
+/**
+ * @primitive b.mailStore.fts.columnAndFieldFor
+ * @signature b.mailStore.fts.columnAndFieldFor(filterKey)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Map a search filter key (`subject` / `body` / `from` / `to`) to
+ * the FTS5 column it indexes into PLUS the namespace pseudo-field
+ * the indexer uses when hashing tokens. Used by the search path so
+ * the query-side hash transform matches the index-side one byte-
+ * for-byte.
+ *
+ * @example
+ *   b.mailStore.fts.columnAndFieldFor("from");
+ *   // → { column: "addr_toks", field: "addr" }
+ */
+function columnAndFieldFor(key) {
+  return QUERY_KEY_MAP[key] || null;
+}
+
+// Rewrite an operator query term into a FTS5 MATCH expression. The
+// term is tokenized + hashed exactly like an index value, then the
+// hashes are AND'd together so multi-word queries require every
+// token to appear in the row. Returns null when no tokens survive
+// the filter (caller should skip the FTS join in that case).
+/**
+ * @primitive b.mailStore.fts.buildMatchExpression
+ * @signature b.mailStore.fts.buildMatchExpression(table, field, term)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Tokenize + hash an operator's query `term` and produce the FTS5
+ * MATCH expression that selects rows containing every surviving
+ * token. Returns `null` when no tokens survive the tokenize +
+ * stopword filter (caller skips the FTS join in that case).
+ *
+ * @example
+ *   var expr = b.mailStore.fts.buildMatchExpression("t", "body", "kubernetes deploy");
+ *   // → "<16hex> AND <16hex>"
+ */
+function buildMatchExpression(table, field, term) {
+  var tokens = tokenize(term);
+  if (tokens.length === 0) return null;
+  var hashes = [];
+  var seen = Object.create(null);
+  for (var i = 0; i < tokens.length; i++) {
+    var h = hashToken(table, field, tokens[i]);
+    if (!h || seen[h]) continue;
+    seen[h] = true;
+    hashes.push(h);
+  }
+  if (hashes.length === 0) return null;
+  // FTS5 default operator is AND; explicit for readability.
+  return hashes.join(" AND ");
+}
+
+// SQL builder — creates the FTS5 virtual table. Caller supplies the
+// quoted parent table identifier; this module owns the FTS table
+// name and column layout.
+/**
+ * @primitive b.mailStore.fts.createSql
+ * @signature b.mailStore.fts.createSql(qFtsTable)
+ * @since     0.11.25
+ * @status    stable
+ *
+ * Returns the `CREATE VIRTUAL TABLE IF NOT EXISTS` SQL for the
+ * sealed-token FTS5 table. The caller passes the quoted table
+ * identifier (e.g. `"blamejs_mail_messages_fts"`).
+ *
+ * @example
+ *   db.prepare(b.mailStore.fts.createSql('"mail_fts"')).run();
+ */
+function createSql(qFtsTable) {
+  return "CREATE VIRTUAL TABLE IF NOT EXISTS " + qFtsTable + " USING fts5(" +
+    "objectid UNINDEXED, " +
+    "subject_toks, " +
+    "addr_toks, " +
+    "body_toks, " +
+    "tokenize = 'unicode61 remove_diacritics 2'" +
+  ")";
+}
+
+module.exports = {
+  // SQL primitives
+  createSql:           createSql,
+
+  // Index-side
+  tokenize:            tokenize,
+  hashToken:           hashToken,
+  hashTokens:          hashTokens,
+  hashText:            hashText,
+  rowFromMessage:      rowFromMessage,
+
+  // Query-side
+  buildMatchExpression: buildMatchExpression,
+  columnAndFieldFor:   columnAndFieldFor,
+  QUERY_KEY_MAP:       QUERY_KEY_MAP,
+
+  // Constants surfaced for tests + adjacent modules.
+  STOPWORDS:           STOPWORDS,
+  MIN_TOKEN_LEN:       MIN_TOKEN_LEN,
+  MAX_TOKEN_LEN:       MAX_TOKEN_LEN,
+  MAX_TOKENS_PER_FIELD: MAX_TOKENS_PER_FIELD,
+  FTS_FIELDS:          FTS_FIELDS,
+};

package/lib/mail-store.js

@@ -63,6 +63,7 @@ var cryptoField = require("./crypto-field");
 var safeMime = require("./safe-mime");
 var safeSql = require("./safe-sql");
 var guardMessageId = require("./guard-message-id");
+var mailStoreFts = require("./mail-store-fts");
 var { defineClass } = require("./framework-error");
 
 var MailStoreError = defineClass("MailStoreError", { alwaysPermanent: true });
@@ -126,6 +127,7 @@ function create(opts) {
   var qFolders = safeSql.quoteIdentifier(prefix + "_folders",   "sqlite");
   var qFlags   = safeSql.quoteIdentifier(prefix + "_flags",     "sqlite");
   var qQuota   = safeSql.quoteIdentifier(prefix + "_quota",     "sqlite");
+  var qFts     = safeSql.quoteIdentifier(prefix + "_messages_fts", "sqlite");
   var messagesTable = prefix + "_messages";
 
   var maxMessageBytes = opts.maxMessageBytes !== undefined ? opts.maxMessageBytes : DEFAULT_MAX_MESSAGE_BYTES;
@@ -147,7 +149,7 @@ function create(opts) {
   });
 
   if (doInit) {
-    _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota);
+    _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota, qFts);
     _ensureDefaultFolders(db, qFolders);
   }
 
@@ -195,12 +197,27 @@ function create(opts) {
     " WHERE folder_id = ? AND objectid IN (SELECT value FROM json_each(?))");
   var stmtDeleteMsg        = db.prepare("DELETE FROM " + qMsgs + " WHERE objectid = ?");
   var stmtDeleteFlags      = db.prepare("DELETE FROM " + qFlags + " WHERE objectid = ?");
+  // Sealed-token FTS5 prepared statements — index sync runs in the
+  // same transaction window as the canonical row mutation so a crash
+  // between the two cannot leave the FTS index out of step with the
+  // messages table. See lib/mail-store-fts.js for the tokenize +
+  // vault-salted-hash transform applied here.
+  var stmtInsertFts        = db.prepare(
+    "INSERT INTO " + qFts + " (objectid, subject_toks, addr_toks, body_toks) VALUES (?, ?, ?, ?)");
+  var stmtDeleteFts        = db.prepare("DELETE FROM " + qFts + " WHERE objectid = ?");
 
   return {
     appendMessage:    function (folderName, rawBytes, appendOpts) {
-      return _appendMessage({
+      // Wrap canonical row insert + FTS row insert in a single backend
+      // transaction so a crash / FTS-row failure CANNOT leave a message
+      // persisted but unsearchable (state drift). better-sqlite3-style
+      // backends expose `.transaction(fn)()`; backends without
+      // transactions fall back to per-statement (the FTS insert is the
+      // last write, so partial state == still consistent to the reader).
+      var args = {
         db: db, qMsgs: qMsgs, qFlags: qFlags, messagesTable: messagesTable,
         stmtInsertMsg: stmtInsertMsg,
+        stmtInsertFts: stmtInsertFts,
         stmtBumpFolderModseq: stmtBumpFolderModseq,
         stmtGetFolderByName: stmtGetFolderByName,
         stmtFindThreadByMsgId: stmtFindThreadByMsgId,
@@ -209,7 +226,13 @@ function create(opts) {
         safeMimeOpts: safeMimeOpts,
         maxMessageBytes: maxMessageBytes,
         maxBodyBytes: maxBodyBytes,
-      });
+      };
+      if (typeof db.transaction === "function") {
+        var result;
+        db.transaction(function () { result = _appendMessage(args); })();
+        return result;
+      }
+      return _appendMessage(args);
     },
     fetchByObjectId:  function (folderName, objectid) {
       return _fetchByObjectId({
@@ -220,6 +243,97 @@ function create(opts) {
         folderName: folderName, objectid: objectid,
       });
     },
+    /**
+     * search — sealed-token full-text search inside a single folder.
+     *
+     * Composes the FTS5 virtual table populated by `appendMessage`.
+     * Each filter term is tokenized + vault-salted-hashed exactly like
+     * the index side, then issued as an FTS5 `MATCH` expression
+     * intersected with the modseq + flag window. Result rows carry the
+     * SAME shape as `queryByModseq` so operators iterate either path
+     * symmetrically.
+     *
+     * `filter` accepts (any subset; all present terms AND-combine):
+     *   - text:        match across subject + addr + body
+     *   - subject:     match against `subject_toks` column only
+     *   - body:        match against `body_toks` column only
+     *   - from / to:   match against `addr_toks`
+     *   - sinceModseq: integer floor
+     *   - limit:       result cap (default 100, hard cap 1000)
+     *
+     * When NO text-side filter is present, falls through to the
+     * `queryByModseq` path — search is purely additive on the existing
+     * modseq cursor.
+     */
+    search:           function (folderName, filter) {
+      var folder = stmtGetFolderByName.get(folderName);
+      if (!folder) {
+        throw new MailStoreError("mail-store/no-folder",
+          "search: folder '" + folderName + "' not found");
+      }
+      var f = filter || {};
+      var sinceModseq = f.sinceModseq || 0;
+      var limit = f.limit || 100;
+      if (limit > 1000) limit = 1000;                                                                  // allow:raw-byte-literal — query row cap, not bytes
+
+      var matchClauses = [];
+      function addMatch(filterKey, term) {
+        if (!term) return;
+        var m = mailStoreFts.columnAndFieldFor(filterKey);
+        if (!m) return;
+        var expr = mailStoreFts.buildMatchExpression(messagesTable, m.field, term);
+        if (expr) matchClauses.push(m.column + ":(" + expr + ")");
+      }
+      if (f.subject) addMatch("subject", f.subject);
+      if (f.body)    addMatch("body",    f.body);
+      if (f.from)    addMatch("from",    f.from);
+      if (f.to)      addMatch("to",      f.to);
+      if (f.text) {
+        var perCol = ["subject", "body", "from"].map(function (key) {
+          var m = mailStoreFts.columnAndFieldFor(key);
+          var perColExpr = mailStoreFts.buildMatchExpression(messagesTable, m.field, f.text);
+          return perColExpr ? "(" + m.column + ":(" + perColExpr + "))" : null;
+        }).filter(Boolean);
+        if (perCol.length > 0) {
+          matchClauses.push("(" + perCol.join(" OR ") + ")");
+        }
+      }
+
+      if (matchClauses.length === 0) {
+        var fallback = stmtQueryByModseq.all(folder.id, sinceModseq, limit);
+        return {
+          rows: fallback.map(function (r) {
+            return {
+              objectid: r.objectid, modseq: r.modseq, sizeBytes: r.size_bytes,
+              internalDate: r.internal_date, legalHold: r.legal_hold === 1,
+            };
+          }),
+          nextModseq: fallback.length > 0 ? fallback[fallback.length - 1].modseq : sinceModseq,
+        };
+      }
+
+      var matchExpr = matchClauses.join(" AND ");
+      // FTS5 MATCH binds to the virtual-table name — aliases / joined-
+      // table refs are parsed as ordinary column refs and fail. The
+      // IN-subquery shape sidesteps that.
+      var sql =
+        "SELECT objectid, modseq, size_bytes, internal_date, legal_hold " +
+        "FROM " + qMsgs + " " +
+        "WHERE folder_id = ? AND modseq > ? " +
+        "AND objectid IN (SELECT objectid FROM " + qFts + " WHERE " + qFts + " MATCH ?) " +
+        "ORDER BY modseq ASC LIMIT ?";
+      var rows = db.prepare(sql).all(folder.id, sinceModseq, matchExpr, limit);
+      return {
+        rows: rows.map(function (r) {
+          return {
+            objectid: r.objectid, modseq: r.modseq, sizeBytes: r.size_bytes,
+            internalDate: r.internal_date, legalHold: r.legal_hold === 1,
+          };
+        }),
+        nextModseq: rows.length > 0 ? rows[rows.length - 1].modseq : sinceModseq,
+        matchExpr: matchExpr,
+      };
+    },
     queryByModseq:    function (folderName, queryOpts) {
       var folder = stmtGetFolderByName.get(folderName);
       if (!folder) {
@@ -368,6 +482,7 @@ function create(opts) {
       function _runTxn() {
         for (var di = 0; di < toDelete.length; di += 1) {
           stmtDeleteFlags.run(toDelete[di].objectid);
+          stmtDeleteFts.run(toDelete[di].objectid);
           stmtDeleteMsg.run(toDelete[di].objectid);
           totalBytes += toDelete[di].size_bytes || 0;
         }
@@ -512,6 +627,18 @@ function _appendMessage(args) {
   args.stmtBumpFolderModseq.run(modseq, args.folderName);
   args.stmtBumpQuota.run(folder.id, buf.length, 1);
 
+  // FTS index update — tokenize the PRE-seal plaintext, hash each
+  // token with the per-deployment vault salt, insert into the FTS5
+  // virtual table.
+  var ftsRow = mailStoreFts.rowFromMessage(args.messagesTable, {
+    objectid: objectid,
+    subject:  subject,
+    from:     fromAddr,
+    to:       toAddrs,
+    body:     bodyText,
+  });
+  args.stmtInsertFts.run(ftsRow.objectid, ftsRow.subject_toks, ftsRow.addr_toks, ftsRow.body_toks);
+
   return { objectid: objectid, modseq: modseq, sizeBytes: buf.length, threadRootId: threadRootId };
 }
 
@@ -706,7 +833,7 @@ function _normalizeMsgId(s) {
 
 // ---- Schema bootstrap ----------------------------------------------------
 
-function _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota) {
+function _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota, qFts) {
   // Folders table — created first since messages reference folder_id.
   db.prepare(
     "CREATE TABLE IF NOT EXISTS " + qFolders + " (" +
@@ -775,6 +902,13 @@ function _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota) {
     "cap_count INTEGER, " +
     "FOREIGN KEY(folder_id) REFERENCES " + qFolders + "(id))"
   ).run();
+
+  // Sealed-token FTS5 virtual table. The token-hash transform lives in
+  // `lib/mail-store-fts.js`; this is the storage layer. Tokenizer is
+  // `unicode61 remove_diacritics 2` so FTS5's segmenter splits hash-
+  // tokens on whitespace exactly — hashes are ASCII-hex-only, so no
+  // Unicode case-fold runs at MATCH time.
+  db.prepare(mailStoreFts.createSql(qFts)).run();
 }
 
 function _ensureDefaultFolders(db, qFolders) {
@@ -788,4 +922,8 @@ module.exports = {
   create:             create,
   DEFAULT_FOLDERS:    DEFAULT_FOLDERS,
   MailStoreError:     MailStoreError,
+  // Sealed-token FTS substrate. Exposed for adjacent primitives (e.g.
+  // wire-protocol adapters translating IMAP SEARCH TEXT into the
+  // store's FTS5 column expression).
+  fts:                mailStoreFts,
 };