@blamejs/blamejs-shop 0.3.35 → 0.3.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -0
- package/lib/admin.js +375 -1
- package/lib/asset-manifest.json +1 -1
- package/lib/storefront.js +450 -0
- package/lib/vendor/MANIFEST.json +3 -3
- package/lib/vendor/blamejs/CHANGELOG.md +2 -0
- package/lib/vendor/blamejs/api-snapshot.json +14 -2
- package/lib/vendor/blamejs/lib/crypto-field.js +69 -0
- package/lib/vendor/blamejs/lib/mail-store-fts.js +40 -18
- package/lib/vendor/blamejs/lib/mail-store.js +188 -2
- package/lib/vendor/blamejs/package.json +1 -1
- package/lib/vendor/blamejs/release-notes/v0.14.10.json +54 -0
- package/lib/vendor/blamejs/test/layer-0-primitives/codebase-patterns.test.js +17 -6
- package/lib/vendor/blamejs/test/layer-0-primitives/crypto-field-derived-hash.test.js +44 -0
- package/lib/vendor/blamejs/test/layer-0-primitives/mail-store-fts.test.js +208 -0
- package/package.json +1 -1
|
@@ -54,10 +54,23 @@
|
|
|
54
54
|
* nothing readable; search works against ciphertext.
|
|
55
55
|
*/
|
|
56
56
|
|
|
57
|
-
var
|
|
58
|
-
var vault = require("./vault");
|
|
57
|
+
var cryptoField = require("./crypto-field");
|
|
59
58
|
var C = require("./constants");
|
|
60
59
|
|
|
60
|
+
// Sealed-token FTS on-disk format version. Bumped when the token-hash
|
|
61
|
+
// transform changes so the mail-store reindex path can detect a stale
|
|
62
|
+
// index and rebuild it from the sealed messages table. v1 was the
|
|
63
|
+
// legacy salted-sha3-truncated hand-roll; v2 is the keyed
|
|
64
|
+
// hmac-shake256 digest computed via cryptoField.computeNamespacedHash.
|
|
65
|
+
var FTS_FORMAT_VERSION = 2;
|
|
66
|
+
|
|
67
|
+
// Per-token hash width, in bytes. 8 bytes -> 16 hex chars. Full 64-char
|
|
68
|
+
// SHA3 / SHAKE digest is overkill for the FTS hash space, and the
|
|
69
|
+
// shorter token compresses the FTS5 row 4x without observable collision
|
|
70
|
+
// risk at corpus sizes the framework targets (<= 10^9 unique tokens,
|
|
71
|
+
// where 64-bit collision space leaves the birthday bound > 10^9).
|
|
72
|
+
var FTS_HASH_BYTES = 8;
|
|
73
|
+
|
|
61
74
|
// Stopwords are dropped before hashing — they'd dominate every row's
|
|
62
75
|
// token set without adding query selectivity. Kept conservative to
|
|
63
76
|
// stay locale-neutral for v1.
|
|
@@ -154,26 +167,29 @@ function tokenize(text) {
|
|
|
154
167
|
return out;
|
|
155
168
|
}
|
|
156
169
|
|
|
157
|
-
// Hash one token
|
|
158
|
-
//
|
|
170
|
+
// Hash one token through the canonical cryptoField primitive
|
|
171
|
+
// (computeNamespacedHash) so the FTS index inherits the keyed-MAC
|
|
172
|
+
// digest used for derived-hash mirrors on sealed columns. The
|
|
159
173
|
// namespace is per-table, per-field, per-purpose ("fts") so that
|
|
160
|
-
// rotating an operator's vault
|
|
161
|
-
//
|
|
162
|
-
// — full 64-char
|
|
163
|
-
// tokens compress the FTS5 row 4x without
|
|
164
|
-
// at corpus sizes the framework targets
|
|
165
|
-
// 64-bit collision space leaves the
|
|
174
|
+
// rotating an operator's vault key invalidates every FTS row in the
|
|
175
|
+
// same step as every sealed column. Returns a 16-char hex prefix
|
|
176
|
+
// (FTS_HASH_BYTES bytes) — full 64-char digest is overkill for the
|
|
177
|
+
// FTS hash space, and shorter tokens compress the FTS5 row 4x without
|
|
178
|
+
// observable collision risk at corpus sizes the framework targets
|
|
179
|
+
// (<= 10^9 unique tokens, where 64-bit collision space leaves the
|
|
180
|
+
// birthday bound > 10^9).
|
|
166
181
|
/**
|
|
167
182
|
* @primitive b.mailStore.fts.hashToken
|
|
168
183
|
* @signature b.mailStore.fts.hashToken(table, field, token)
|
|
169
184
|
* @since 0.11.25
|
|
170
185
|
* @status stable
|
|
171
186
|
*
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
* Returns a 16-char hex
|
|
187
|
+
* Keyed hash of one token under the (table, field) namespace. Routes
|
|
188
|
+
* through `b.cryptoField.computeNamespacedHash` in `hmac-shake256`
|
|
189
|
+
* mode — the same keyed-MAC machinery that protects sealed-column
|
|
190
|
+
* derived hashes — so rotating the vault key invalidates every FTS
|
|
191
|
+
* hash in step with every sealed-column hash. Returns a 16-char hex
|
|
192
|
+
* prefix.
|
|
177
193
|
*
|
|
178
194
|
* @example
|
|
179
195
|
* var h = b.mailStore.fts.hashToken("mail_messages", "body", "kubernetes");
|
|
@@ -185,9 +201,10 @@ function hashToken(table, field, token) {
|
|
|
185
201
|
// fields are pseudo-fields (no sealed-column registration), so the
|
|
186
202
|
// canonical fallback path is always the right answer here.
|
|
187
203
|
var ns = "bj-" + table + "-" + field + ":fts:";
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
204
|
+
return cryptoField.computeNamespacedHash(ns, token, {
|
|
205
|
+
mode: "hmac-shake256",
|
|
206
|
+
truncateBytes: FTS_HASH_BYTES,
|
|
207
|
+
});
|
|
191
208
|
}
|
|
192
209
|
|
|
193
210
|
// Hash a token array → space-separated string suitable for FTS5
|
|
@@ -391,4 +408,9 @@ module.exports = {
|
|
|
391
408
|
MAX_TOKEN_LEN: MAX_TOKEN_LEN,
|
|
392
409
|
MAX_TOKENS_PER_FIELD: MAX_TOKENS_PER_FIELD,
|
|
393
410
|
FTS_FIELDS: FTS_FIELDS,
|
|
411
|
+
FTS_HASH_BYTES: FTS_HASH_BYTES,
|
|
412
|
+
// On-disk format marker — the mail-store reindex path stamps this
|
|
413
|
+
// into `<prefix>_meta` once a full rebuild completes; a stale/missing
|
|
414
|
+
// marker triggers a rebuild from the sealed messages table.
|
|
415
|
+
FTS_FORMAT_VERSION: FTS_FORMAT_VERSION,
|
|
394
416
|
};
|
|
@@ -60,6 +60,7 @@
|
|
|
60
60
|
var C = require("./constants");
|
|
61
61
|
var bCrypto = require("./crypto");
|
|
62
62
|
var cryptoField = require("./crypto-field");
|
|
63
|
+
var vault = require("./vault");
|
|
63
64
|
var safeMime = require("./safe-mime");
|
|
64
65
|
var safeSql = require("./safe-sql");
|
|
65
66
|
var guardMessageId = require("./guard-message-id");
|
|
@@ -72,6 +73,14 @@ var DEFAULT_TABLE_PREFIX = "blamejs_mail";
|
|
|
72
73
|
var DEFAULT_MAX_MESSAGE_BYTES = C.BYTES.mib(50);
|
|
73
74
|
var DEFAULT_MAX_BODY_BYTES = C.BYTES.mib(25);
|
|
74
75
|
|
|
76
|
+
// `<prefix>_meta` marker key carrying the FTS on-disk format version.
|
|
77
|
+
// The reindex path writes a `'rebuilding'` sentinel BEFORE clearing the
|
|
78
|
+
// index and the final format-version string only AFTER every row is
|
|
79
|
+
// reinserted, so a partial / interrupted rebuild is never marked
|
|
80
|
+
// complete and never queried as a finished index.
|
|
81
|
+
var FTS_FORMAT_META_KEY = "fts_format";
|
|
82
|
+
var FTS_REBUILDING_SENTINEL = "rebuilding";
|
|
83
|
+
|
|
75
84
|
// Standard IMAP4rev2 default folders + JMAP role mapping.
|
|
76
85
|
var DEFAULT_FOLDERS = Object.freeze([
|
|
77
86
|
{ name: "INBOX", role: "inbox" },
|
|
@@ -129,6 +138,7 @@ function create(opts) {
|
|
|
129
138
|
var qFlags = safeSql.quoteIdentifier(prefix + "_flags", "sqlite");
|
|
130
139
|
var qQuota = safeSql.quoteIdentifier(prefix + "_quota", "sqlite");
|
|
131
140
|
var qFts = safeSql.quoteIdentifier(prefix + "_messages_fts", "sqlite");
|
|
141
|
+
var qMeta = safeSql.quoteIdentifier(prefix + "_meta", "sqlite");
|
|
132
142
|
var messagesTable = prefix + "_messages";
|
|
133
143
|
|
|
134
144
|
var maxMessageBytes = opts.maxMessageBytes !== undefined ? opts.maxMessageBytes : DEFAULT_MAX_MESSAGE_BYTES;
|
|
@@ -150,7 +160,7 @@ function create(opts) {
|
|
|
150
160
|
});
|
|
151
161
|
|
|
152
162
|
if (doInit) {
|
|
153
|
-
_ensureSchema(db, qMsgs, qFolders, qFlags, qQuota, qFts);
|
|
163
|
+
_ensureSchema(db, qMsgs, qFolders, qFlags, qQuota, qFts, qMeta);
|
|
154
164
|
_ensureDefaultFolders(db, qFolders);
|
|
155
165
|
}
|
|
156
166
|
|
|
@@ -207,6 +217,150 @@ function create(opts) {
|
|
|
207
217
|
"INSERT INTO " + qFts + " (objectid, subject_toks, addr_toks, body_toks) VALUES (?, ?, ?, ?)");
|
|
208
218
|
var stmtDeleteFts = db.prepare("DELETE FROM " + qFts + " WHERE objectid = ?");
|
|
209
219
|
|
|
220
|
+
// `<prefix>_meta` marker read — used by the reindex gate at create()
|
|
221
|
+
// AND by every FTS query path (search) to refuse a half-built index.
|
|
222
|
+
// Guarded behind a closure so a store opened with init:false against a
|
|
223
|
+
// pre-format-version database (no _meta table) reads as "no marker"
|
|
224
|
+
// instead of throwing.
|
|
225
|
+
function _readFtsMarker() {
|
|
226
|
+
try {
|
|
227
|
+
var row = db.prepare("SELECT value FROM " + qMeta + " WHERE key = ?")
|
|
228
|
+
.get(FTS_FORMAT_META_KEY);
|
|
229
|
+
return row ? row.value : null;
|
|
230
|
+
} catch (_e) {
|
|
231
|
+
// _meta table absent (init:false against a legacy store) — treat
|
|
232
|
+
// as no marker so the FTS query path falls back rather than
|
|
233
|
+
// querying a possibly-stale index as if it were current.
|
|
234
|
+
return null;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
// The FTS index is queryable only when the on-disk marker equals the
|
|
238
|
+
// current format version. A `'rebuilding'` sentinel or any other
|
|
239
|
+
// value (stale format, missing marker) means the index is non-final;
|
|
240
|
+
// search() falls back to the modseq cursor rather than returning
|
|
241
|
+
// partial / mixed-scheme FTS hits.
|
|
242
|
+
var CURRENT_FTS_FMT = String(mailStoreFts.FTS_FORMAT_VERSION);
|
|
243
|
+
function _ftsIndexUsable() {
|
|
244
|
+
return _readFtsMarker() === CURRENT_FTS_FMT;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// Reindex-on-upgrade. Runs once at create() (when doInit) AFTER the
|
|
248
|
+
// schema is ensured. Rebuilds the sealed-token FTS index from the
|
|
249
|
+
// canonical (sealed) messages table whenever the on-disk format
|
|
250
|
+
// marker is missing or stale — the token-hash transform changed, so
|
|
251
|
+
// every previously-indexed row hashes to a different value and the
|
|
252
|
+
// old rows are unsearchable under the new scheme.
|
|
253
|
+
//
|
|
254
|
+
// Data-safety contract:
|
|
255
|
+
// - A `'rebuilding'` sentinel is written BEFORE the DELETE, so an
|
|
256
|
+
// interrupted rebuild leaves a non-final marker; search() refuses
|
|
257
|
+
// the index until a later create() completes the rebuild.
|
|
258
|
+
// - The DELETE + every reinsert run inside an explicit
|
|
259
|
+
// BEGIN/COMMIT/ROLLBACK (ordinary SQLite — works on node:sqlite
|
|
260
|
+
// and b.db alike; neither exposes a usable `.transaction(fn)()`
|
|
261
|
+
// for this shape). A throw inside the insert loop rolls the whole
|
|
262
|
+
// rebuild back, leaving the OLD index intact + queryable... except
|
|
263
|
+
// the marker still reads as non-final, so search falls back until
|
|
264
|
+
// the next successful create() — the index is never silently
|
|
265
|
+
// half-built.
|
|
266
|
+
// - The final format-version marker is written ONLY after every row
|
|
267
|
+
// reinserts and the COMMIT lands.
|
|
268
|
+
function _reindexFts() {
|
|
269
|
+
var fmt = _readFtsMarker();
|
|
270
|
+
if (fmt === CURRENT_FTS_FMT) return { reindexed: false, reason: "current" };
|
|
271
|
+
|
|
272
|
+
// Count existing FTS rows AFTER the early-return so the common
|
|
273
|
+
// already-current path pays nothing for the scan.
|
|
274
|
+
var ftsCountRow = db.prepare("SELECT COUNT(*) AS n FROM " + qFts).get();
|
|
275
|
+
var ftsCount = (ftsCountRow && ftsCountRow.n) || 0;
|
|
276
|
+
var msgCountRow = db.prepare("SELECT COUNT(*) AS n FROM " + qMsgs).get();
|
|
277
|
+
var msgCount = (msgCountRow && msgCountRow.n) || 0;
|
|
278
|
+
|
|
279
|
+
// Fresh store — no marker AND nothing indexed AND no messages to
|
|
280
|
+
// index. Just stamp the current format; there is nothing to rebuild.
|
|
281
|
+
if (fmt === null && ftsCount === 0 && msgCount === 0) {
|
|
282
|
+
_writeFtsMarker(CURRENT_FTS_FMT);
|
|
283
|
+
return { reindexed: false, reason: "fresh" };
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// Reindex unseals EVERY message row — a runtime dependency on the
|
|
287
|
+
// vault that a bare create() otherwise wouldn't have. Fail loud at
|
|
288
|
+
// boot (entry-point tier) rather than silently leaving a stale,
|
|
289
|
+
// wrong-scheme index searchable: an operator who constructs the
|
|
290
|
+
// store before vault.init() must see the misordering immediately.
|
|
291
|
+
if (!vault.isInitialized()) {
|
|
292
|
+
throw new MailStoreError("mail-store/fts-reindex-vault-uninitialized",
|
|
293
|
+
"mailStore.create: FTS index format is stale (marker=" +
|
|
294
|
+
JSON.stringify(fmt) + ", current=" + CURRENT_FTS_FMT + ") and a " +
|
|
295
|
+
"reindex from the sealed messages table requires the vault - call " +
|
|
296
|
+
"b.vault.init(...) BEFORE b.mailStore.create(...). Refusing to leave " +
|
|
297
|
+
"a stale, wrong-scheme search index queryable.");
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Sentinel BEFORE any destructive work — a crash between here and
|
|
301
|
+
// the final marker leaves the index marked non-final.
|
|
302
|
+
_writeFtsMarker(FTS_REBUILDING_SENTINEL);
|
|
303
|
+
|
|
304
|
+
// BEGIN IMMEDIATE takes the write lock up front so the message
|
|
305
|
+
// snapshot, the FTS delete, and the reinsert are one isolated window.
|
|
306
|
+
// The snapshot is read INSIDE the lock: a concurrent writer in another
|
|
307
|
+
// process cannot append a message between the snapshot and the delete
|
|
308
|
+
// (which would otherwise drop that row's freshly-inserted FTS entry
|
|
309
|
+
// without re-adding it, silently losing it from search).
|
|
310
|
+
var allRows;
|
|
311
|
+
db.prepare("BEGIN IMMEDIATE").run();
|
|
312
|
+
try {
|
|
313
|
+
allRows = db.prepare("SELECT * FROM " + qMsgs).all();
|
|
314
|
+
db.prepare("DELETE FROM " + qFts).run();
|
|
315
|
+
for (var i = 0; i < allRows.length; i += 1) {
|
|
316
|
+
// unsealRow returns the DB column names (subject / from_addr /
|
|
317
|
+
// to_addrs / body_text); map them into the FTS param shape
|
|
318
|
+
// (subject / from / to / body) rowFromMessage expects.
|
|
319
|
+
var clear = cryptoField.unsealRow(messagesTable, allRows[i]);
|
|
320
|
+
var ftsRow = mailStoreFts.rowFromMessage(messagesTable, {
|
|
321
|
+
objectid: clear.objectid,
|
|
322
|
+
subject: clear.subject || "",
|
|
323
|
+
from: clear.from_addr || "",
|
|
324
|
+
to: clear.to_addrs || "",
|
|
325
|
+
body: clear.body_text || "",
|
|
326
|
+
});
|
|
327
|
+
stmtInsertFts.run(ftsRow.objectid, ftsRow.subject_toks,
|
|
328
|
+
ftsRow.addr_toks, ftsRow.body_toks);
|
|
329
|
+
}
|
|
330
|
+
db.prepare("COMMIT").run();
|
|
331
|
+
} catch (e) {
|
|
332
|
+
// Roll the partial rebuild back — the OLD index rows are restored
|
|
333
|
+
// (the DELETE is undone), so the prior scheme stays intact and
|
|
334
|
+
// queryable. The marker still reads as the sentinel, so search()
|
|
335
|
+
// falls back until a later create() completes the rebuild: a
|
|
336
|
+
// retriable, never-silently-half-built state.
|
|
337
|
+
try { db.prepare("ROLLBACK").run(); } catch (_re) { /* best-effort */ }
|
|
338
|
+
throw new MailStoreError("mail-store/fts-reindex-failed",
|
|
339
|
+
"mailStore.create: FTS reindex from the sealed messages table " +
|
|
340
|
+
"failed and was rolled back (the prior index is intact); retry " +
|
|
341
|
+
"after resolving: " + ((e && e.message) || String(e)));
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// Full rebuild committed — stamp the current format LAST so the
|
|
345
|
+
// index only reads as final once every row is present.
|
|
346
|
+
_writeFtsMarker(CURRENT_FTS_FMT);
|
|
347
|
+
return { reindexed: true, rows: allRows.length };
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function _writeFtsMarker(value) {
|
|
351
|
+
db.prepare(
|
|
352
|
+
"INSERT INTO " + qMeta + " (key, value) VALUES (?, ?) " +
|
|
353
|
+
"ON CONFLICT(key) DO UPDATE SET value = excluded.value"
|
|
354
|
+
).run(FTS_FORMAT_META_KEY, value);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// Wire the reindex into create() AFTER schema bootstrap. A fresh store
|
|
358
|
+
// (no marker, 0 FTS rows, 0 messages) just stamps the format; a store
|
|
359
|
+
// carrying an old-format index rebuilds from the sealed rows.
|
|
360
|
+
if (doInit) {
|
|
361
|
+
_reindexFts();
|
|
362
|
+
}
|
|
363
|
+
|
|
210
364
|
return {
|
|
211
365
|
appendMessage: function (folderName, rawBytes, appendOpts) {
|
|
212
366
|
// Wrap canonical row insert + FTS row insert in a single backend
|
|
@@ -277,6 +431,27 @@ function create(opts) {
|
|
|
277
431
|
var limit = f.limit || 100;
|
|
278
432
|
if (limit > 1000) limit = 1000; // query row cap, not bytes
|
|
279
433
|
|
|
434
|
+
// The FTS index is queryable only when the on-disk format marker
|
|
435
|
+
// is current. A `'rebuilding'` sentinel (mid-rebuild / interrupted
|
|
436
|
+
// rebuild) or a stale/missing marker means the index is non-final
|
|
437
|
+
// — fall back to the bare modseq cursor rather than returning
|
|
438
|
+
// partial or wrong-scheme FTS hits. Returning a subset of the true
|
|
439
|
+
// matches as if it were complete is the corruption hazard the
|
|
440
|
+
// reindex sentinel exists to prevent.
|
|
441
|
+
if (!_ftsIndexUsable()) {
|
|
442
|
+
var nonFinal = stmtQueryByModseq.all(folder.id, sinceModseq, limit);
|
|
443
|
+
return {
|
|
444
|
+
rows: nonFinal.map(function (r) {
|
|
445
|
+
return {
|
|
446
|
+
objectid: r.objectid, modseq: r.modseq, sizeBytes: r.size_bytes,
|
|
447
|
+
internalDate: r.internal_date, legalHold: r.legal_hold === 1,
|
|
448
|
+
};
|
|
449
|
+
}),
|
|
450
|
+
nextModseq: nonFinal.length > 0 ? nonFinal[nonFinal.length - 1].modseq : sinceModseq,
|
|
451
|
+
ftsUnavailable: true,
|
|
452
|
+
};
|
|
453
|
+
}
|
|
454
|
+
|
|
280
455
|
var matchClauses = [];
|
|
281
456
|
function addMatch(filterKey, term) {
|
|
282
457
|
if (!term) return;
|
|
@@ -834,7 +1009,7 @@ function _normalizeMsgId(s) {
|
|
|
834
1009
|
|
|
835
1010
|
// ---- Schema bootstrap ----------------------------------------------------
|
|
836
1011
|
|
|
837
|
-
function _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota, qFts) {
|
|
1012
|
+
function _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota, qFts, qMeta) {
|
|
838
1013
|
// Folders table — created first since messages reference folder_id.
|
|
839
1014
|
db.prepare(
|
|
840
1015
|
"CREATE TABLE IF NOT EXISTS " + qFolders + " (" +
|
|
@@ -910,6 +1085,17 @@ function _ensureSchema(db, qMsgs, qFolders, qFlags, qQuota, qFts) {
|
|
|
910
1085
|
// tokens on whitespace exactly — hashes are ASCII-hex-only, so no
|
|
911
1086
|
// Unicode case-fold runs at MATCH time.
|
|
912
1087
|
db.prepare(mailStoreFts.createSql(qFts)).run();
|
|
1088
|
+
|
|
1089
|
+
// Per-prefix key/value metadata table. Holds the FTS on-disk format
|
|
1090
|
+
// marker (`fts_format`) so the reindex path can detect a stale index
|
|
1091
|
+
// and rebuild it. Scoped per table-prefix (NOT PRAGMA user_version,
|
|
1092
|
+
// which is db-global and would collide across stores sharing one
|
|
1093
|
+
// sqlite file).
|
|
1094
|
+
db.prepare(
|
|
1095
|
+
"CREATE TABLE IF NOT EXISTS " + qMeta + " (" +
|
|
1096
|
+
"key TEXT PRIMARY KEY, " +
|
|
1097
|
+
"value TEXT)"
|
|
1098
|
+
).run();
|
|
913
1099
|
}
|
|
914
1100
|
|
|
915
1101
|
function _ensureDefaultFolders(db, qFolders) {
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "../scripts/release-notes-schema.json",
|
|
3
|
+
"version": "0.14.10",
|
|
4
|
+
"date": "2026-05-31",
|
|
5
|
+
"headline": "Full-text-search token hashes move to a keyed MAC; existing mail-store search indexes rebuild automatically on upgrade",
|
|
6
|
+
"summary": "The mail-store full-text-search index hashed its tokens with a hand-rolled salted-SHA3 derived hash. It now routes through the framework's sealed-column hashing primitive in keyed mode (HMAC-SHAKE256 off the per-deployment MAC key), so a search-index token hash is unforgeable and un-correlatable across deployments without that key — the same posture the sealed-column lookup hashes already use. Because the keyed hash changes the stored token values, a mail-store opened after upgrade detects its index as old-format and rebuilds it once from the sealed message rows. The rebuild runs under a format marker: the index is marked `rebuilding` before it is cleared and only marked current after every row is re-hashed inside an explicit transaction, and search falls back to its cursor path (rather than returning partial hits) whenever the marker is not current — so an interrupted rebuild leaves the old index intact and queryable and retries on the next open, never serving a half-built index. A new `b.cryptoField.computeNamespacedHash` primitive backs the keyed hashing for callers that hash outside the registered-column path.",
|
|
7
|
+
"sections": [
|
|
8
|
+
{
|
|
9
|
+
"heading": "Added",
|
|
10
|
+
"items": [
|
|
11
|
+
{
|
|
12
|
+
"title": "`b.cryptoField.computeNamespacedHash`",
|
|
13
|
+
"body": "A mode-aware namespaced hash for indexed-lookup callers that hash a value outside the registered-column derived-hash path. `computeNamespacedHash(ns, value, { mode, truncateBytes })` routes through the same engine as `computeDerived` — `salted-sha3` (default) or the keyed `hmac-shake256` — with optional hex truncation. The mail-store full-text index is the first consumer."
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"heading": "Changed",
|
|
19
|
+
"items": [
|
|
20
|
+
{
|
|
21
|
+
"title": "Mail-store full-text index rehashes to a keyed MAC on upgrade",
|
|
22
|
+
"body": "The full-text-search token hash now uses `b.cryptoField.computeNamespacedHash` in `hmac-shake256` mode instead of a hand-rolled salted-SHA3. The first time a store is opened after upgrade, its index is detected as old-format and rebuilt once from the sealed message rows; subsequent opens are no-ops. Search is unaffected once the rebuild completes. The rebuild requires the vault to be initialized and fails closed (a clear error) at construction if it is not, rather than leaving a stale searchable index."
|
|
23
|
+
}
|
|
24
|
+
]
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"heading": "Security",
|
|
28
|
+
"items": [
|
|
29
|
+
{
|
|
30
|
+
"title": "Keyed, un-correlatable full-text-search token hashes",
|
|
31
|
+
"body": "A search-index token hash is now a keyed MAC over a per-deployment key, not a static-salted digest — it cannot be forged or correlated across deployments without that key, closing the low-entropy-token correlation gap on the search index. The index remains unrecoverable from a database dump alone, as before."
|
|
32
|
+
}
|
|
33
|
+
]
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"heading": "Detectors",
|
|
37
|
+
"items": [
|
|
38
|
+
{
|
|
39
|
+
"title": "Hand-rolled lookup-hash check covers the split form",
|
|
40
|
+
"body": "The check that requires sealed-column lookup hashes to compose the framework primitive now also catches the across-lines hand-roll (`var salt = getDerivedHashSalt(); var hex = salt.toString(...); sha3(hex + ns + value)`), not only the single-expression form, so the bypass that the mail-store index used can't reappear."
|
|
41
|
+
}
|
|
42
|
+
]
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"heading": "Migration",
|
|
46
|
+
"items": [
|
|
47
|
+
{
|
|
48
|
+
"title": "Automatic, one-time full-text index rebuild",
|
|
49
|
+
"body": "No operator action is required: the rebuild runs automatically and idempotently on first open after upgrade, atomically and crash-safe (an interrupted rebuild keeps the old index and retries). The only requirement is that the vault is initialized before the mail-store is constructed. One caveat for shared stores: do not run a pre-upgrade and post-upgrade node against the same backend file concurrently across this format change — the old node would write old-format hashes the new node cannot match. Roll the deployment fully across the upgrade. This re-open condition is lifted once all nodes are on 0.14.10 or later."
|
|
50
|
+
}
|
|
51
|
+
]
|
|
52
|
+
}
|
|
53
|
+
]
|
|
54
|
+
}
|
|
@@ -8195,18 +8195,29 @@ var KNOWN_ANTIPATTERNS = [
|
|
|
8195
8195
|
// and risks a static-salt regression if the salt source is changed
|
|
8196
8196
|
// for one caller but not the canonical helper.
|
|
8197
8197
|
//
|
|
8198
|
-
// Detector:
|
|
8199
|
-
//
|
|
8200
|
-
//
|
|
8198
|
+
// Detector: hand-rolled `sha3Hash(<salt-hex> + ns + value)` outside
|
|
8199
|
+
// crypto-field.js (the canonical helper). Two shapes are caught:
|
|
8200
|
+
// (a) inline — `getDerivedHashSalt().toString("hex") + ...`
|
|
8201
|
+
// (b) split across lines — a variable bound to the salt's hex is
|
|
8202
|
+
// concatenated into a `sha3Hash(...)`:
|
|
8203
|
+
// var salt = getDerivedHashSalt();
|
|
8204
|
+
// var h = salt.toString("hex");
|
|
8205
|
+
// sha3Hash(h + ns + v);
|
|
8206
|
+
// Shape (b) evaded the original inline-only regex (the salt's
|
|
8207
|
+
// `.toString("hex")` is followed by `;`, not `+`, and the `+` lives
|
|
8208
|
+
// on the downstream sha3Hash argument). The helper itself is
|
|
8209
|
+
// allowlisted. NOTE: a `getDerivedHashSalt().toString("hex")` fed
|
|
8210
|
+
// into `kdf(...)` (e.g. idempotency-key's fingerprint HMAC seed) is
|
|
8211
|
+
// NOT this bug and does not match — neither shape names `sha3Hash`.
|
|
8201
8212
|
id: "derived-hash-handrolled-outside-crypto-field",
|
|
8202
|
-
primitive: "cryptoField derived/lookup hashes — compute via _computeDerivedHash (honours salted-sha3 vs hmac-shake256 mode + vault.getDerivedHashMacKey); do not hand-roll sha3Hash(getDerivedHashSalt() + ns + value) at call sites",
|
|
8203
|
-
regex: /getDerivedHashSalt\s*\(\s*\)\s*\.toString\s*\(\s*["']hex["']\s*\)\s*\+/,
|
|
8213
|
+
primitive: "cryptoField derived/lookup hashes — compute via _computeDerivedHash / b.cryptoField.computeNamespacedHash (honours salted-sha3 vs hmac-shake256 mode + vault.getDerivedHashMacKey); do not hand-roll sha3Hash(getDerivedHashSalt() + ns + value) at call sites",
|
|
8214
|
+
regex: /getDerivedHashSalt\s*\(\s*\)\s*\.toString\s*\(\s*["']hex["']\s*\)\s*\+|var\s+(\w+)\s*=\s*[\w.]*getDerivedHashSalt\s*\(\s*\)\s*;[\s\S]{0,200}?\b(\w+)\s*=\s*\1\s*\.toString\s*\(\s*["']hex["']\s*\)\s*;[\s\S]{0,200}?sha3Hash\s*\(\s*\2\s*\+/,
|
|
8204
8215
|
skipCommentLines: true,
|
|
8205
8216
|
allowlist: [
|
|
8206
8217
|
// The canonical helper — _computeDerivedHash branches on mode here.
|
|
8207
8218
|
"lib/crypto-field.js",
|
|
8208
8219
|
],
|
|
8209
|
-
reason: "v0.14.7 — derived-hash equality lookups gained a keyed mode (hmac-shake256 off vault.getDerivedHashMacKey) alongside the salted-sha3 default. The mode decision lives in cryptoField._computeDerivedHash. A site that hand-derives the hash with getDerivedHashSalt() bypasses the keyed option and the per-column mode policy; only the canonical helper (crypto-field.js) may name the salt directly.",
|
|
8220
|
+
reason: "v0.14.7 — derived-hash equality lookups gained a keyed mode (hmac-shake256 off vault.getDerivedHashMacKey) alongside the salted-sha3 default. The mode decision lives in cryptoField._computeDerivedHash (wrapped by the public computeNamespacedHash for pseudo-field indexes such as the mail-store FTS tokens). A site that hand-derives the hash with getDerivedHashSalt() — inline OR split across local variables feeding sha3Hash — bypasses the keyed option and the per-column mode policy; only the canonical helper (crypto-field.js) may name the salt directly.",
|
|
8210
8221
|
},
|
|
8211
8222
|
|
|
8212
8223
|
{
|
|
@@ -88,6 +88,50 @@ async function run() {
|
|
|
88
88
|
} catch (e) { badColMode = /mode must be/.test(e.message); }
|
|
89
89
|
check("bad per-column mode throws at registerTable", badColMode);
|
|
90
90
|
|
|
91
|
+
// ---- computeNamespacedHash: pseudo-field indexed hash (FTS substrate) ----
|
|
92
|
+
// Wraps _computeDerivedHash for namespaces that have no registered
|
|
93
|
+
// derived-hash column (e.g. the sealed-token mail-store FTS index).
|
|
94
|
+
var ns = "bj-mail_messages-body:fts:";
|
|
95
|
+
|
|
96
|
+
// Default mode is salted-sha3 → byte-identical to the legacy hand-roll.
|
|
97
|
+
var saltHex = b.vault.getDerivedHashSalt().toString("hex");
|
|
98
|
+
var legacyRef = b.crypto.sha3Hash(saltHex + ns + "kubernetes");
|
|
99
|
+
var cnhDefault = b.cryptoField.computeNamespacedHash(ns, "kubernetes");
|
|
100
|
+
check("computeNamespacedHash default is salted-sha3 (128 hex)", cnhDefault.length === 128);
|
|
101
|
+
check("computeNamespacedHash default == legacy hand-roll", cnhDefault === legacyRef);
|
|
102
|
+
|
|
103
|
+
// truncateBytes slices the hex to that many bytes (2 hex chars each).
|
|
104
|
+
var cnhTrunc = b.cryptoField.computeNamespacedHash(ns, "kubernetes",
|
|
105
|
+
{ mode: "salted-sha3", truncateBytes: 8 });
|
|
106
|
+
check("computeNamespacedHash truncateBytes:8 -> 16 hex", cnhTrunc.length === 16);
|
|
107
|
+
check("computeNamespacedHash truncate is a prefix of full", legacyRef.slice(0, 16) === cnhTrunc);
|
|
108
|
+
|
|
109
|
+
// Keyed mode differs from the salted default and is unforgeable
|
|
110
|
+
// without the MAC key.
|
|
111
|
+
var cnhKeyed = b.cryptoField.computeNamespacedHash(ns, "kubernetes",
|
|
112
|
+
{ mode: "hmac-shake256", truncateBytes: 8 });
|
|
113
|
+
check("computeNamespacedHash hmac-shake256 -> 16 hex", /^[0-9a-f]{16}$/.test(cnhKeyed));
|
|
114
|
+
check("computeNamespacedHash keyed differs from salted", cnhKeyed !== cnhTrunc);
|
|
115
|
+
check("computeNamespacedHash is deterministic",
|
|
116
|
+
cnhKeyed === b.cryptoField.computeNamespacedHash(ns, "kubernetes",
|
|
117
|
+
{ mode: "hmac-shake256", truncateBytes: 8 }));
|
|
118
|
+
|
|
119
|
+
// Config-time throws: bad mode, non-positive / non-integer truncateBytes.
|
|
120
|
+
var cnhBadMode = false;
|
|
121
|
+
try { b.cryptoField.computeNamespacedHash(ns, "x", { mode: "md5" }); }
|
|
122
|
+
catch (e) { cnhBadMode = /mode must be/.test(e.message); }
|
|
123
|
+
check("computeNamespacedHash bad mode throws", cnhBadMode);
|
|
124
|
+
|
|
125
|
+
var cnhBadTruncZero = false;
|
|
126
|
+
try { b.cryptoField.computeNamespacedHash(ns, "x", { truncateBytes: 0 }); }
|
|
127
|
+
catch (e) { cnhBadTruncZero = /truncateBytes must be/.test(e.message); }
|
|
128
|
+
check("computeNamespacedHash truncateBytes:0 throws", cnhBadTruncZero);
|
|
129
|
+
|
|
130
|
+
var cnhBadTruncFloat = false;
|
|
131
|
+
try { b.cryptoField.computeNamespacedHash(ns, "x", { truncateBytes: 2.5 }); }
|
|
132
|
+
catch (e) { cnhBadTruncFloat = /truncateBytes must be/.test(e.message); }
|
|
133
|
+
check("computeNamespacedHash non-integer truncateBytes throws", cnhBadTruncFloat);
|
|
134
|
+
|
|
91
135
|
console.log("OK — crypto-field derived-hash tests");
|
|
92
136
|
}
|
|
93
137
|
|