@blamejs/core 0.11.24 → 0.11.26

package/lib/mail-store.js

@@ -63,6 +63,7 @@ var cryptoField = require("./crypto-field");
 var safeMime = require("./safe-mime");
 var safeSql = require("./safe-sql");
 var guardMessageId = require("./guard-message-id");
+var mailStoreFts = require("./mail-store-fts");
 var { defineClass } = require("./framework-error");
 
 var MailStoreError = defineClass("MailStoreError", { alwaysPermanent: true });
@@ -126,6 +127,7 @@ function create(opts) {
   var qFolders = safeSql.quoteIdentifier(prefix + "_folders",   "sqlite");
   var qFlags   = safeSql.quoteIdentifier(prefix + "_flags",     "sqlite");
   var qQuota   = safeSql.quoteIdentifier(prefix + "_quota",     "sqlite");
+  var qFts     = safeSql.quoteIdentifier(prefix + "_messages_fts", "sqlite");
   var messagesTable = prefix + "_messages";
 
   var maxMessageBytes = opts.maxMessageBytes !== undefined ? opts.maxMessageBytes : DEFAULT_MAX_MESSAGE_BYTES;
@@ -147,7 +149,7 @@ function create(opts) {
   });
 
   if (doInit) {
-    _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota);
+    _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota, qFts);
     _ensureDefaultFolders(db, qFolders);
   }
 
@@ -195,12 +197,27 @@ function create(opts) {
     " WHERE folder_id = ? AND objectid IN (SELECT value FROM json_each(?))");
   var stmtDeleteMsg        = db.prepare("DELETE FROM " + qMsgs + " WHERE objectid = ?");
   var stmtDeleteFlags      = db.prepare("DELETE FROM " + qFlags + " WHERE objectid = ?");
+  // Sealed-token FTS5 prepared statements — index sync runs in the
+  // same transaction window as the canonical row mutation so a crash
+  // between the two cannot leave the FTS index out of step with the
+  // messages table. See lib/mail-store-fts.js for the tokenize +
+  // vault-salted-hash transform applied here.
+  var stmtInsertFts        = db.prepare(
+    "INSERT INTO " + qFts + " (objectid, subject_toks, addr_toks, body_toks) VALUES (?, ?, ?, ?)");
+  var stmtDeleteFts        = db.prepare("DELETE FROM " + qFts + " WHERE objectid = ?");
 
   return {
     appendMessage:    function (folderName, rawBytes, appendOpts) {
-      return _appendMessage({
+      // Wrap canonical row insert + FTS row insert in a single backend
+      // transaction so a crash / FTS-row failure CANNOT leave a message
+      // persisted but unsearchable (state drift). better-sqlite3-style
+      // backends expose `.transaction(fn)()`; backends without
+      // transactions fall back to per-statement (the FTS insert is the
+      // last write, so partial state == still consistent to the reader).
+      var args = {
         db: db, qMsgs: qMsgs, qFlags: qFlags, messagesTable: messagesTable,
         stmtInsertMsg: stmtInsertMsg,
+        stmtInsertFts: stmtInsertFts,
         stmtBumpFolderModseq: stmtBumpFolderModseq,
         stmtGetFolderByName: stmtGetFolderByName,
         stmtFindThreadByMsgId: stmtFindThreadByMsgId,
@@ -209,7 +226,13 @@ function create(opts) {
         safeMimeOpts: safeMimeOpts,
         maxMessageBytes: maxMessageBytes,
         maxBodyBytes: maxBodyBytes,
-      });
+      };
+      if (typeof db.transaction === "function") {
+        var result;
+        db.transaction(function () { result = _appendMessage(args); })();
+        return result;
+      }
+      return _appendMessage(args);
     },
     fetchByObjectId:  function (folderName, objectid) {
       return _fetchByObjectId({
@@ -220,6 +243,97 @@ function create(opts) {
         folderName: folderName, objectid: objectid,
       });
     },
+    /**
+     * search — sealed-token full-text search inside a single folder.
+     *
+     * Composes the FTS5 virtual table populated by `appendMessage`.
+     * Each filter term is tokenized + vault-salted-hashed exactly like
+     * the index side, then issued as an FTS5 `MATCH` expression
+     * intersected with the modseq + flag window. Result rows carry the
+     * SAME shape as `queryByModseq` so operators iterate either path
+     * symmetrically.
+     *
+     * `filter` accepts (any subset; all present terms AND-combine):
+     *   - text:        match across subject + addr + body
+     *   - subject:     match against `subject_toks` column only
+     *   - body:        match against `body_toks` column only
+     *   - from / to:   match against `addr_toks`
+     *   - sinceModseq: integer floor
+     *   - limit:       result cap (default 100, hard cap 1000)
+     *
+     * When NO text-side filter is present, falls through to the
+     * `queryByModseq` path — search is purely additive on the existing
+     * modseq cursor.
+     */
+    search:           function (folderName, filter) {
+      var folder = stmtGetFolderByName.get(folderName);
+      if (!folder) {
+        throw new MailStoreError("mail-store/no-folder",
+          "search: folder '" + folderName + "' not found");
+      }
+      var f = filter || {};
+      var sinceModseq = f.sinceModseq || 0;
+      var limit = f.limit || 100;
+      if (limit > 1000) limit = 1000;                                                                  // allow:raw-byte-literal — query row cap, not bytes
+
+      var matchClauses = [];
+      function addMatch(filterKey, term) {
+        if (!term) return;
+        var m = mailStoreFts.columnAndFieldFor(filterKey);
+        if (!m) return;
+        var expr = mailStoreFts.buildMatchExpression(messagesTable, m.field, term);
+        if (expr) matchClauses.push(m.column + ":(" + expr + ")");
+      }
+      if (f.subject) addMatch("subject", f.subject);
+      if (f.body)    addMatch("body",    f.body);
+      if (f.from)    addMatch("from",    f.from);
+      if (f.to)      addMatch("to",      f.to);
+      if (f.text) {
+        var perCol = ["subject", "body", "from"].map(function (key) {
+          var m = mailStoreFts.columnAndFieldFor(key);
+          var perColExpr = mailStoreFts.buildMatchExpression(messagesTable, m.field, f.text);
+          return perColExpr ? "(" + m.column + ":(" + perColExpr + "))" : null;
+        }).filter(Boolean);
+        if (perCol.length > 0) {
+          matchClauses.push("(" + perCol.join(" OR ") + ")");
+        }
+      }
+
+      if (matchClauses.length === 0) {
+        var fallback = stmtQueryByModseq.all(folder.id, sinceModseq, limit);
+        return {
+          rows: fallback.map(function (r) {
+            return {
+              objectid: r.objectid, modseq: r.modseq, sizeBytes: r.size_bytes,
+              internalDate: r.internal_date, legalHold: r.legal_hold === 1,
+            };
+          }),
+          nextModseq: fallback.length > 0 ? fallback[fallback.length - 1].modseq : sinceModseq,
+        };
+      }
+
+      var matchExpr = matchClauses.join(" AND ");
+      // FTS5 MATCH binds to the virtual-table name — aliases / joined-
+      // table refs are parsed as ordinary column refs and fail. The
+      // IN-subquery shape sidesteps that.
+      var sql =
+        "SELECT objectid, modseq, size_bytes, internal_date, legal_hold " +
+        "FROM " + qMsgs + " " +
+        "WHERE folder_id = ? AND modseq > ? " +
+        "AND objectid IN (SELECT objectid FROM " + qFts + " WHERE " + qFts + " MATCH ?) " +
+        "ORDER BY modseq ASC LIMIT ?";
+      var rows = db.prepare(sql).all(folder.id, sinceModseq, matchExpr, limit);
+      return {
+        rows: rows.map(function (r) {
+          return {
+            objectid: r.objectid, modseq: r.modseq, sizeBytes: r.size_bytes,
+            internalDate: r.internal_date, legalHold: r.legal_hold === 1,
+          };
+        }),
+        nextModseq: rows.length > 0 ? rows[rows.length - 1].modseq : sinceModseq,
+        matchExpr: matchExpr,
+      };
+    },
     queryByModseq:    function (folderName, queryOpts) {
       var folder = stmtGetFolderByName.get(folderName);
       if (!folder) {
@@ -368,6 +482,7 @@ function create(opts) {
       function _runTxn() {
         for (var di = 0; di < toDelete.length; di += 1) {
           stmtDeleteFlags.run(toDelete[di].objectid);
+          stmtDeleteFts.run(toDelete[di].objectid);
           stmtDeleteMsg.run(toDelete[di].objectid);
           totalBytes += toDelete[di].size_bytes || 0;
         }
@@ -512,6 +627,18 @@ function _appendMessage(args) {
   args.stmtBumpFolderModseq.run(modseq, args.folderName);
   args.stmtBumpQuota.run(folder.id, buf.length, 1);
 
+  // FTS index update — tokenize the PRE-seal plaintext, hash each
+  // token with the per-deployment vault salt, insert into the FTS5
+  // virtual table.
+  var ftsRow = mailStoreFts.rowFromMessage(args.messagesTable, {
+    objectid: objectid,
+    subject:  subject,
+    from:     fromAddr,
+    to:       toAddrs,
+    body:     bodyText,
+  });
+  args.stmtInsertFts.run(ftsRow.objectid, ftsRow.subject_toks, ftsRow.addr_toks, ftsRow.body_toks);
+
   return { objectid: objectid, modseq: modseq, sizeBytes: buf.length, threadRootId: threadRootId };
 }
 
@@ -706,7 +833,7 @@ function _normalizeMsgId(s) {
 
 // ---- Schema bootstrap ----------------------------------------------------
 
-function _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota) {
+function _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota, qFts) {
   // Folders table — created first since messages reference folder_id.
   db.prepare(
     "CREATE TABLE IF NOT EXISTS " + qFolders + " (" +
@@ -775,6 +902,13 @@ function _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota) {
     "cap_count INTEGER, " +
     "FOREIGN KEY(folder_id) REFERENCES " + qFolders + "(id))"
   ).run();
+
+  // Sealed-token FTS5 virtual table. The token-hash transform lives in
+  // `lib/mail-store-fts.js`; this is the storage layer. Tokenizer is
+  // `unicode61 remove_diacritics 2` so FTS5's segmenter splits hash-
+  // tokens on whitespace exactly — hashes are ASCII-hex-only, so no
+  // Unicode case-fold runs at MATCH time.
+  db.prepare(mailStoreFts.createSql(qFts)).run();
 }
 
 function _ensureDefaultFolders(db, qFolders) {
@@ -788,4 +922,8 @@ module.exports = {
   create:             create,
   DEFAULT_FOLDERS:    DEFAULT_FOLDERS,
   MailStoreError:     MailStoreError,
+  // Sealed-token FTS substrate. Exposed for adjacent primitives (e.g.
+  // wire-protocol adapters translating IMAP SEARCH TEXT into the
+  // store's FTS5 column expression).
+  fts:                mailStoreFts,
 };