@blamejs/core 0.14.27 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/README.md +2 -2
- package/index.js +4 -0
- package/lib/ai-content-detect.js +9 -10
- package/lib/api-key.js +107 -74
- package/lib/atomic-file.js +29 -1
- package/lib/audit-chain.js +47 -11
- package/lib/audit-sign.js +77 -2
- package/lib/audit-tools.js +79 -51
- package/lib/audit.js +218 -100
- package/lib/backup/index.js +13 -10
- package/lib/break-glass.js +202 -144
- package/lib/cache.js +174 -105
- package/lib/chain-writer.js +38 -16
- package/lib/cli.js +19 -14
- package/lib/cluster-provider-db.js +130 -104
- package/lib/cluster-storage.js +119 -22
- package/lib/cluster.js +119 -71
- package/lib/compliance.js +22 -0
- package/lib/consent.js +73 -24
- package/lib/constants.js +16 -11
- package/lib/crypto-field.js +387 -91
- package/lib/db-declare-row-policy.js +35 -22
- package/lib/db-file-lifecycle.js +3 -2
- package/lib/db-query.js +497 -255
- package/lib/db-schema.js +209 -44
- package/lib/db.js +176 -95
- package/lib/external-db-migrate.js +229 -139
- package/lib/external-db.js +25 -15
- package/lib/framework-error.js +11 -0
- package/lib/framework-files.js +73 -0
- package/lib/framework-schema.js +695 -394
- package/lib/gate-contract.js +596 -1
- package/lib/guard-agent-registry.js +26 -44
- package/lib/guard-all.js +1 -0
- package/lib/guard-auth.js +42 -112
- package/lib/guard-cidr.js +33 -154
- package/lib/guard-csv.js +46 -113
- package/lib/guard-domain.js +34 -157
- package/lib/guard-dsn.js +27 -43
- package/lib/guard-email.js +47 -69
- package/lib/guard-envelope.js +19 -32
- package/lib/guard-event-bus-payload.js +24 -42
- package/lib/guard-event-bus-topic.js +25 -43
- package/lib/guard-filename.js +42 -106
- package/lib/guard-graphql.js +42 -123
- package/lib/guard-html.js +53 -108
- package/lib/guard-idempotency-key.js +24 -42
- package/lib/guard-image.js +46 -103
- package/lib/guard-imap-command.js +18 -32
- package/lib/guard-jmap.js +16 -30
- package/lib/guard-json.js +38 -108
- package/lib/guard-jsonpath.js +38 -171
- package/lib/guard-jwt.js +49 -179
- package/lib/guard-list-id.js +25 -41
- package/lib/guard-list-unsubscribe.js +27 -43
- package/lib/guard-mail-compose.js +24 -42
- package/lib/guard-mail-move.js +26 -44
- package/lib/guard-mail-query.js +28 -46
- package/lib/guard-mail-reply.js +24 -42
- package/lib/guard-mail-sieve.js +24 -42
- package/lib/guard-managesieve-command.js +17 -31
- package/lib/guard-markdown.js +37 -104
- package/lib/guard-message-id.js +26 -45
- package/lib/guard-mime.js +39 -151
- package/lib/guard-oauth.js +54 -135
- package/lib/guard-pdf.js +45 -101
- package/lib/guard-pop3-command.js +21 -31
- package/lib/guard-posture-chain.js +24 -42
- package/lib/guard-regex.js +33 -107
- package/lib/guard-saga-config.js +24 -42
- package/lib/guard-shell.js +42 -172
- package/lib/guard-smtp-command.js +48 -54
- package/lib/guard-snapshot-envelope.js +24 -42
- package/lib/guard-sql.js +1491 -0
- package/lib/guard-stream-args.js +24 -43
- package/lib/guard-svg.js +47 -65
- package/lib/guard-template.js +35 -172
- package/lib/guard-tenant-id.js +26 -45
- package/lib/guard-time.js +32 -154
- package/lib/guard-trace-context.js +25 -44
- package/lib/guard-uuid.js +32 -153
- package/lib/guard-xml.js +38 -113
- package/lib/guard-yaml.js +51 -163
- package/lib/http-client.js +14 -0
- package/lib/inbox.js +120 -107
- package/lib/legal-hold.js +107 -50
- package/lib/log-stream-cloudwatch.js +47 -31
- package/lib/log-stream-otlp.js +32 -18
- package/lib/mail-crypto-smime.js +2 -6
- package/lib/mail-greylist.js +2 -6
- package/lib/mail-helo.js +2 -6
- package/lib/mail-journal.js +85 -64
- package/lib/mail-rbl.js +2 -6
- package/lib/mail-scan.js +2 -6
- package/lib/mail-spam-score.js +2 -6
- package/lib/mail-store.js +287 -154
- package/lib/middleware/fetch-metadata.js +17 -7
- package/lib/middleware/idempotency-key.js +54 -38
- package/lib/middleware/rate-limit.js +102 -32
- package/lib/middleware/security-headers.js +21 -5
- package/lib/migrations.js +108 -66
- package/lib/network-heartbeat.js +7 -0
- package/lib/nonce-store.js +31 -9
- package/lib/object-store/azure-blob-bucket-ops.js +9 -4
- package/lib/object-store/azure-blob.js +31 -3
- package/lib/object-store/sigv4.js +10 -0
- package/lib/outbox.js +136 -82
- package/lib/pqc-agent.js +44 -0
- package/lib/pubsub-cluster.js +42 -20
- package/lib/queue-local.js +202 -139
- package/lib/queue-redis.js +9 -1
- package/lib/queue-sqs.js +6 -0
- package/lib/retention.js +82 -39
- package/lib/safe-dns.js +29 -45
- package/lib/safe-ical.js +18 -33
- package/lib/safe-icap.js +27 -43
- package/lib/safe-sieve.js +21 -40
- package/lib/safe-sql.js +124 -3
- package/lib/safe-vcard.js +18 -33
- package/lib/scheduler.js +35 -12
- package/lib/seeders.js +122 -74
- package/lib/session-stores.js +42 -14
- package/lib/session.js +109 -72
- package/lib/sql.js +3885 -0
- package/lib/static.js +45 -7
- package/lib/subject.js +55 -17
- package/lib/vault/index.js +3 -2
- package/lib/vault/passphrase-ops.js +3 -2
- package/lib/vault/rotate.js +104 -64
- package/lib/vendor-data.js +2 -0
- package/lib/websocket.js +16 -0
- package/package.json +1 -1
- package/sbom.cdx.json +6 -6
package/lib/mail-store.js
CHANGED
|
@@ -63,6 +63,7 @@ var cryptoField = require("./crypto-field");
|
|
|
63
63
|
var vault = require("./vault");
|
|
64
64
|
var safeMime = require("./safe-mime");
|
|
65
65
|
var safeSql = require("./safe-sql");
|
|
66
|
+
var sql = require("./sql");
|
|
66
67
|
var guardMessageId = require("./guard-message-id");
|
|
67
68
|
var mailStoreFts = require("./mail-store-fts");
|
|
68
69
|
var { defineClass } = require("./framework-error");
|
|
@@ -133,13 +134,28 @@ function create(opts) {
|
|
|
133
134
|
throw new MailStoreError("mail-store/bad-table-prefix",
|
|
134
135
|
"mailStore.create: tablePrefix is not a valid SQL identifier: " + e.message);
|
|
135
136
|
}
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
137
|
+
// Bare logical table names. The store targets a CONCRETE sqlite handle
|
|
138
|
+
// (the operator's backend), never b.clusterStorage, so every b.sql
|
|
139
|
+
// builder is constructed with { dialect: "sqlite", quoteName: true } so
|
|
140
|
+
// the prefixed name emits as a quoted `"..."` identifier (the same
|
|
141
|
+
// quote-by-construction the prior hand-rolled safeSql.quoteIdentifier
|
|
142
|
+
// produced). The few quoted tokens kept below feed the two spots a verb
|
|
143
|
+
// builder cannot reach: the FTS5 virtual-table self-reference in a MATCH
|
|
144
|
+
// sub-query and the existing-row self-reference inside the quota upsert's
|
|
145
|
+
// `col = col + EXCLUDED.col` expression.
|
|
142
146
|
var messagesTable = prefix + "_messages";
|
|
147
|
+
var foldersTable = prefix + "_folders";
|
|
148
|
+
var flagsTable = prefix + "_flags";
|
|
149
|
+
var quotaTable = prefix + "_quota";
|
|
150
|
+
var ftsTable = prefix + "_messages_fts";
|
|
151
|
+
var metaTable = prefix + "_meta";
|
|
152
|
+
var SQL = { dialect: "sqlite", quoteName: true };
|
|
153
|
+
// The only quoted table token kept as a literal: the quota table's
|
|
154
|
+
// existing-row self-reference inside the `col = col +/- ...` SET
|
|
155
|
+
// expressions (the decrement update + the accumulate-on-conflict upsert),
|
|
156
|
+
// which a structured set()/value cell cannot express. Every other table
|
|
157
|
+
// identifier emits through the b.sql builders above with quoteName.
|
|
158
|
+
var qQuota = safeSql.quoteIdentifier(quotaTable, "sqlite");
|
|
143
159
|
|
|
144
160
|
var maxMessageBytes = opts.maxMessageBytes !== undefined ? opts.maxMessageBytes : DEFAULT_MAX_MESSAGE_BYTES;
|
|
145
161
|
var maxBodyBytes = opts.maxBodyBytes !== undefined ? opts.maxBodyBytes : DEFAULT_MAX_BODY_BYTES;
|
|
@@ -160,62 +176,138 @@ function create(opts) {
|
|
|
160
176
|
});
|
|
161
177
|
|
|
162
178
|
if (doInit) {
|
|
163
|
-
_ensureSchema(db,
|
|
164
|
-
|
|
179
|
+
_ensureSchema(db, {
|
|
180
|
+
messagesTable: messagesTable, foldersTable: foldersTable,
|
|
181
|
+
flagsTable: flagsTable, quotaTable: quotaTable,
|
|
182
|
+
ftsTable: ftsTable, metaTable: metaTable,
|
|
183
|
+
});
|
|
184
|
+
_ensureDefaultFolders(db, foldersTable);
|
|
165
185
|
}
|
|
166
186
|
|
|
167
|
-
// Prepared statements — cached across the store lifetime.
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
)
|
|
175
|
-
|
|
176
|
-
var
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
"
|
|
191
|
-
var
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
187
|
+
// Prepared statements — cached across the store lifetime. Every
|
|
188
|
+
// statement text is composed through b.sql (sqlite dialect, quoteName so
|
|
189
|
+
// the prefixed table name emits as a quoted identifier against the
|
|
190
|
+
// concrete handle) and then prepared once; the node:sqlite handle binds
|
|
191
|
+
// the `?` placeholders positionally at run() / get() / all() time. Every
|
|
192
|
+
// VALUES / SET cell is a `"?"` placeholder string so the builder emits a
|
|
193
|
+
// `?` per column (only the SQL TEXT is kept; the throwaway params array
|
|
194
|
+
// of literal "?" strings is discarded) — the caller binds the real values
|
|
195
|
+
// at run() time, including the leading 0 the INSERT seeds `legal_hold` to.
|
|
196
|
+
var stmtInsertMsg = db.prepare(sql.insert(messagesTable, SQL)
|
|
197
|
+
.columns([
|
|
198
|
+
"objectid", "folder_id", "modseq", "internal_date", "received_at",
|
|
199
|
+
"size_bytes", "message_id", "message_id_hash", "in_reply_to",
|
|
200
|
+
"references_csv", "thread_root_id", "subject", "from_addr", "from_hash",
|
|
201
|
+
"to_addrs", "body_text", "body_html", "legal_hold",
|
|
202
|
+
])
|
|
203
|
+
.values([
|
|
204
|
+
"?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?",
|
|
205
|
+
]).toSql().sql);
|
|
206
|
+
var stmtBumpFolderModseq = db.prepare(sql.update(foldersTable, SQL)
|
|
207
|
+
.set("modseq_max", "?").where("name", "?").toSql().sql);
|
|
208
|
+
var stmtGetFolderByName = db.prepare(sql.select(foldersTable, SQL)
|
|
209
|
+
.columns(["id", "name", "role", "parent_id", "modseq_max", "uidvalidity"])
|
|
210
|
+
.where("name", "?").toSql().sql);
|
|
211
|
+
var stmtFetchMsg = db.prepare(sql.select(messagesTable, SQL)
|
|
212
|
+
.where("objectid", "?").where("folder_id", "?").toSql().sql);
|
|
213
|
+
// The modseq-cursor read carries a per-call row LIMIT. b.sql's limit()
|
|
214
|
+
// takes a concrete integer (a bound LIMIT placeholder is not a builder
|
|
215
|
+
// shape), and the limit is a server-controlled, hard-capped integer — so
|
|
216
|
+
// the statement is composed per call with the integer baked in (the
|
|
217
|
+
// folder_id + sinceModseq still bind as `?`). Cached prepared statements
|
|
218
|
+
// are reserved for the fixed-shape reads above.
|
|
219
|
+
function _queryByModseqStmt(limit) {
|
|
220
|
+
// Coerce the per-call limit to a non-negative integer before it reaches
|
|
221
|
+
// b.sql's limit() (which THROWS on a non-integer). The limit is an
|
|
222
|
+
// operator-supplied query opt, so this is the defensive-reader tier: a
|
|
223
|
+
// garbage value floors to a sane page size rather than crashing the read.
|
|
224
|
+
var n = Math.floor(Number(limit));
|
|
225
|
+
if (!isFinite(n) || n < 0) n = 1000;
|
|
226
|
+
return db.prepare(sql.select(messagesTable, SQL)
|
|
227
|
+
.columns(["objectid", "modseq", "size_bytes", "internal_date", "legal_hold"])
|
|
228
|
+
.where("folder_id", "?").whereOp("modseq", ">", "?")
|
|
229
|
+
.orderBy("modseq", "asc").limit(n).toSql().sql);
|
|
230
|
+
}
|
|
231
|
+
function _queryByModseqRows(folderId, sinceModseq, limit) {
|
|
232
|
+
return _queryByModseqStmt(limit).all(folderId, sinceModseq);
|
|
233
|
+
}
|
|
234
|
+
var stmtFlagsForMsg = db.prepare(sql.select(flagsTable, SQL)
|
|
235
|
+
.columns(["flag"]).where("objectid", "?").toSql().sql);
|
|
236
|
+
// INSERT OR IGNORE — composed as ON CONFLICT(objectid, flag) DO NOTHING
|
|
237
|
+
// (the flags PK), the sqlite-final upsert form b.sql emits for the
|
|
238
|
+
// idempotent flag set.
|
|
239
|
+
var stmtSetFlag = db.prepare(sql.upsert(flagsTable, SQL)
|
|
240
|
+
.columns(["objectid", "flag", "set_at"]).values({ objectid: "?", flag: "?", set_at: "?" })
|
|
241
|
+
.onConflict(["objectid", "flag"]).doNothing().toSql().sql);
|
|
242
|
+
var stmtUnsetFlag = db.prepare(sql.delete(flagsTable, SQL)
|
|
243
|
+
.where("objectid", "?").where("flag", "?").toSql().sql);
|
|
244
|
+
var stmtLegalHold = db.prepare(sql.update(messagesTable, SQL)
|
|
245
|
+
.set("legal_hold", "?").where("objectid", "?").toSql().sql);
|
|
246
|
+
var stmtMoveByObjectId = db.prepare(sql.update(messagesTable, SQL)
|
|
247
|
+
.set("folder_id", "?").set("modseq", "?")
|
|
248
|
+
.where("objectid", "?").where("folder_id", "?").toSql().sql);
|
|
249
|
+
var stmtSizeByObjectId = db.prepare(sql.select(messagesTable, SQL)
|
|
250
|
+
.columns(["size_bytes"]).where("objectid", "?").where("folder_id", "?").toSql().sql);
|
|
251
|
+
// `used_bytes = used_bytes - ?` — a guarded setRaw expression (the
|
|
252
|
+
// decrement amount binds as the leading `?`; the column self-reference
|
|
253
|
+
// is quoted by construction).
|
|
254
|
+
var stmtDecrementQuota = db.prepare(sql.update(quotaTable, SQL)
|
|
255
|
+
.setRaw("used_bytes", qQuota + ".\"used_bytes\" - ?", ["?"])
|
|
256
|
+
.setRaw("used_count", qQuota + ".\"used_count\" - ?", ["?"])
|
|
257
|
+
.where("folder_id", "?").toSql().sql);
|
|
258
|
+
var stmtThreadFor = db.prepare(sql.select(messagesTable, SQL)
|
|
259
|
+
.columns(["objectid"]).where("thread_root_id", "?")
|
|
260
|
+
.orderBy("received_at", "asc").toSql().sql);
|
|
261
|
+
var stmtFindThreadByMsgId = db.prepare(sql.select(messagesTable, SQL)
|
|
262
|
+
.columns(["objectid", "thread_root_id"]).where("message_id_hash", "?")
|
|
263
|
+
.limit(1).toSql().sql);
|
|
264
|
+
var stmtInsertFolder = db.prepare(sql.insert(foldersTable, SQL)
|
|
265
|
+
.columns(["name", "role", "parent_id", "modseq_max", "uidvalidity"])
|
|
266
|
+
.values(["?", "?", "?", "?", "?"]).toSql().sql);
|
|
267
|
+
var stmtListFolders = db.prepare(sql.select(foldersTable, SQL)
|
|
268
|
+
.columns(["id", "name", "role", "parent_id", "modseq_max"]).toSql().sql);
|
|
269
|
+
var stmtQuotaForFolder = db.prepare(sql.select(quotaTable, SQL)
|
|
270
|
+
.columns(["used_bytes", "used_count", "cap_bytes", "cap_count"])
|
|
271
|
+
.where("folder_id", "?").toSql().sql);
|
|
272
|
+
// INSERT ... ON CONFLICT(folder_id) DO UPDATE SET col = col + EXCLUDED.col
|
|
273
|
+
// — the atomic per-folder quota accumulator. The existing-row and
|
|
274
|
+
// proposed-row (EXCLUDED) self-references are quoted by construction off
|
|
275
|
+
// the resolved table name so the prefix stays consistent across the
|
|
276
|
+
// INSERT target and the SET expression (the same shape b.rateLimit's
|
|
277
|
+
// window accumulator composes).
|
|
278
|
+
var stmtBumpQuota = db.prepare(sql.upsert(quotaTable, SQL)
|
|
279
|
+
.columns(["folder_id", "used_bytes", "used_count", "cap_bytes", "cap_count"])
|
|
280
|
+
.values({ folder_id: "?", used_bytes: "?", used_count: "?", cap_bytes: "?", cap_count: "?" })
|
|
281
|
+
.onConflict(["folder_id"])
|
|
282
|
+
.doUpdate({
|
|
283
|
+
used_bytes: qQuota + ".\"used_bytes\" + EXCLUDED.\"used_bytes\"",
|
|
284
|
+
used_count: qQuota + ".\"used_count\" + EXCLUDED.\"used_count\"",
|
|
285
|
+
}).toSql().sql);
|
|
201
286
|
// Hard-expunge prepared statements — used by `hardExpunge` to delete
|
|
202
287
|
// a message permanently after retention-floor + legal-hold gates
|
|
203
288
|
// pass. The SELECT is the gate-input source (legal_hold flag + age);
|
|
204
289
|
// the DELETE + flag-cleanup + quota-decrement run inside a backend
|
|
205
|
-
// transaction so partial state can't survive a crash.
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
var
|
|
210
|
-
|
|
290
|
+
// transaction so partial state can't survive a crash. The objectid set
|
|
291
|
+
// arrives as a JSON array string and is unrolled through the sqlite
|
|
292
|
+
// json_each table-valued function (whereInJsonEach) so the candidate
|
|
293
|
+
// list binds as a single `?` rather than a variable placeholder count.
|
|
294
|
+
var stmtSelectForExpunge = db.prepare(sql.select(messagesTable, SQL)
|
|
295
|
+
.columns(["objectid", "folder_id", "size_bytes", "received_at", "legal_hold"])
|
|
296
|
+
.where("folder_id", "?").whereInJsonEach("objectid", "?").toSql().sql);
|
|
297
|
+
var stmtDeleteMsg = db.prepare(sql.delete(messagesTable, SQL)
|
|
298
|
+
.where("objectid", "?").toSql().sql);
|
|
299
|
+
var stmtDeleteFlags = db.prepare(sql.delete(flagsTable, SQL)
|
|
300
|
+
.where("objectid", "?").toSql().sql);
|
|
211
301
|
// Sealed-token FTS5 prepared statements — index sync runs in the
|
|
212
302
|
// same transaction window as the canonical row mutation so a crash
|
|
213
303
|
// between the two cannot leave the FTS index out of step with the
|
|
214
304
|
// messages table. See lib/mail-store-fts.js for the tokenize +
|
|
215
305
|
// vault-salted-hash transform applied here.
|
|
216
|
-
var stmtInsertFts = db.prepare(
|
|
217
|
-
"
|
|
218
|
-
|
|
306
|
+
var stmtInsertFts = db.prepare(sql.insert(ftsTable, SQL)
|
|
307
|
+
.columns(["objectid", "subject_toks", "addr_toks", "body_toks"])
|
|
308
|
+
.values(["?", "?", "?", "?"]).toSql().sql);
|
|
309
|
+
var stmtDeleteFts = db.prepare(sql.delete(ftsTable, SQL)
|
|
310
|
+
.where("objectid", "?").toSql().sql);
|
|
219
311
|
|
|
220
312
|
// `<prefix>_meta` marker read — used by the reindex gate at create()
|
|
221
313
|
// AND by every FTS query path (search) to refuse a half-built index.
|
|
@@ -224,7 +316,8 @@ function create(opts) {
|
|
|
224
316
|
// instead of throwing.
|
|
225
317
|
function _readFtsMarker() {
|
|
226
318
|
try {
|
|
227
|
-
var row = db.prepare(
|
|
319
|
+
var row = db.prepare(sql.select(metaTable, SQL)
|
|
320
|
+
.columns(["value"]).where("key", "?").toSql().sql)
|
|
228
321
|
.get(FTS_FORMAT_META_KEY);
|
|
229
322
|
return row ? row.value : null;
|
|
230
323
|
} catch (_e) {
|
|
@@ -271,9 +364,9 @@ function create(opts) {
|
|
|
271
364
|
|
|
272
365
|
// Count existing FTS rows AFTER the early-return so the common
|
|
273
366
|
// already-current path pays nothing for the scan.
|
|
274
|
-
var ftsCountRow = db.prepare(
|
|
367
|
+
var ftsCountRow = db.prepare(sql.select(ftsTable, SQL).count("*", "n").toSql().sql).get();
|
|
275
368
|
var ftsCount = (ftsCountRow && ftsCountRow.n) || 0;
|
|
276
|
-
var msgCountRow = db.prepare(
|
|
369
|
+
var msgCountRow = db.prepare(sql.select(messagesTable, SQL).count("*", "n").toSql().sql).get();
|
|
277
370
|
var msgCount = (msgCountRow && msgCountRow.n) || 0;
|
|
278
371
|
|
|
279
372
|
// Fresh store — no marker AND nothing indexed AND no messages to
|
|
@@ -310,8 +403,8 @@ function create(opts) {
|
|
|
310
403
|
var allRows;
|
|
311
404
|
db.prepare("BEGIN IMMEDIATE").run();
|
|
312
405
|
try {
|
|
313
|
-
allRows = db.prepare(
|
|
314
|
-
db.prepare(
|
|
406
|
+
allRows = db.prepare(sql.select(messagesTable, SQL).toSql().sql).all();
|
|
407
|
+
db.prepare(sql.delete(ftsTable, SQL).allowNoWhere().toSql().sql).run();
|
|
315
408
|
for (var i = 0; i < allRows.length; i += 1) {
|
|
316
409
|
// unsealRow returns the DB column names (subject / from_addr /
|
|
317
410
|
// to_addrs / body_text); map them into the FTS param shape
|
|
@@ -348,10 +441,10 @@ function create(opts) {
|
|
|
348
441
|
}
|
|
349
442
|
|
|
350
443
|
function _writeFtsMarker(value) {
|
|
351
|
-
db.prepare(
|
|
352
|
-
"
|
|
353
|
-
"
|
|
354
|
-
|
|
444
|
+
db.prepare(sql.upsert(metaTable, SQL)
|
|
445
|
+
.columns(["key", "value"]).values({ key: "?", value: "?" })
|
|
446
|
+
.onConflict(["key"]).doUpdateFromExcluded(["value"]).toSql().sql)
|
|
447
|
+
.run(FTS_FORMAT_META_KEY, value);
|
|
355
448
|
}
|
|
356
449
|
|
|
357
450
|
// Wire the reindex into create() AFTER schema bootstrap. A fresh store
|
|
@@ -370,7 +463,7 @@ function create(opts) {
|
|
|
370
463
|
// transactions fall back to per-statement (the FTS insert is the
|
|
371
464
|
// last write, so partial state == still consistent to the reader).
|
|
372
465
|
var args = {
|
|
373
|
-
db: db,
|
|
466
|
+
db: db, messagesTable: messagesTable,
|
|
374
467
|
stmtInsertMsg: stmtInsertMsg,
|
|
375
468
|
stmtInsertFts: stmtInsertFts,
|
|
376
469
|
stmtBumpFolderModseq: stmtBumpFolderModseq,
|
|
@@ -391,7 +484,7 @@ function create(opts) {
|
|
|
391
484
|
},
|
|
392
485
|
fetchByObjectId: function (folderName, objectid) {
|
|
393
486
|
return _fetchByObjectId({
|
|
394
|
-
db: db,
|
|
487
|
+
db: db, messagesTable: messagesTable,
|
|
395
488
|
stmtGetFolderByName: stmtGetFolderByName,
|
|
396
489
|
stmtFetchMsg: stmtFetchMsg,
|
|
397
490
|
stmtFlagsForMsg: stmtFlagsForMsg,
|
|
@@ -428,7 +521,12 @@ function create(opts) {
|
|
|
428
521
|
}
|
|
429
522
|
var f = filter || {};
|
|
430
523
|
var sinceModseq = f.sinceModseq || 0;
|
|
431
|
-
|
|
524
|
+
// Normalize the row cap to a non-negative integer (default 100, hard
|
|
525
|
+
// cap 1000) before it reaches b.sql's limit() on the FTS path, which
|
|
526
|
+
// THROWS on a non-integer. Defensive-reader tier: a garbage operator
|
|
527
|
+
// value floors to the default page size.
|
|
528
|
+
var limit = Math.floor(Number(f.limit));
|
|
529
|
+
if (!isFinite(limit) || limit < 0) limit = 100;
|
|
432
530
|
if (limit > 1000) limit = 1000; // query row cap, not bytes
|
|
433
531
|
|
|
434
532
|
// The FTS index is queryable only when the on-disk format marker
|
|
@@ -439,7 +537,7 @@ function create(opts) {
|
|
|
439
537
|
// matches as if it were complete is the corruption hazard the
|
|
440
538
|
// reindex sentinel exists to prevent.
|
|
441
539
|
if (!_ftsIndexUsable()) {
|
|
442
|
-
var nonFinal =
|
|
540
|
+
var nonFinal = _queryByModseqRows(folder.id, sinceModseq, limit);
|
|
443
541
|
return {
|
|
444
542
|
rows: nonFinal.map(function (r) {
|
|
445
543
|
return {
|
|
@@ -476,7 +574,7 @@ function create(opts) {
|
|
|
476
574
|
}
|
|
477
575
|
|
|
478
576
|
if (matchClauses.length === 0) {
|
|
479
|
-
var fallback =
|
|
577
|
+
var fallback = _queryByModseqRows(folder.id, sinceModseq, limit);
|
|
480
578
|
return {
|
|
481
579
|
rows: fallback.map(function (r) {
|
|
482
580
|
return {
|
|
@@ -491,14 +589,21 @@ function create(opts) {
|
|
|
491
589
|
var matchExpr = matchClauses.join(" AND ");
|
|
492
590
|
// FTS5 MATCH binds to the virtual-table name — aliases / joined-
|
|
493
591
|
// table refs are parsed as ordinary column refs and fail. The
|
|
494
|
-
// IN-subquery shape sidesteps that
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
var
|
|
592
|
+
// IN-subquery shape (whereIn(col, subBuilder)) sidesteps that: the
|
|
593
|
+
// sub selects objectid from the FTS5 table with `<fts> MATCH ?`
|
|
594
|
+
// (whereMatch, which targets the virtual-table identifier and
|
|
595
|
+
// bypasses the column gate). The MATCH operand stays a bound `?`, so
|
|
596
|
+
// the operator's tokenized-hashed query expression never reshapes the
|
|
597
|
+
// statement. Param bind order: folder_id, sinceModseq, then the
|
|
598
|
+
// sub-query's MATCH operand.
|
|
599
|
+
var ftsSub = sql.select(ftsTable, SQL)
|
|
600
|
+
.columns(["objectid"]).whereMatch(ftsTable, "?");
|
|
601
|
+
var matchStmt = db.prepare(sql.select(messagesTable, SQL)
|
|
602
|
+
.columns(["objectid", "modseq", "size_bytes", "internal_date", "legal_hold"])
|
|
603
|
+
.where("folder_id", "?").whereOp("modseq", ">", "?")
|
|
604
|
+
.whereIn("objectid", ftsSub)
|
|
605
|
+
.orderBy("modseq", "asc").limit(limit).toSql().sql);
|
|
606
|
+
var rows = matchStmt.all(folder.id, sinceModseq, matchExpr);
|
|
502
607
|
return {
|
|
503
608
|
rows: rows.map(function (r) {
|
|
504
609
|
return {
|
|
@@ -518,7 +623,7 @@ function create(opts) {
|
|
|
518
623
|
}
|
|
519
624
|
var sinceModseq = (queryOpts && queryOpts.sinceModseq) || 0;
|
|
520
625
|
var limit = (queryOpts && queryOpts.limit) || 1000; // query row cap, not bytes
|
|
521
|
-
var rows =
|
|
626
|
+
var rows = _queryByModseqRows(folder.id, sinceModseq, limit);
|
|
522
627
|
return rows.map(function (r) {
|
|
523
628
|
return {
|
|
524
629
|
objectid: r.objectid, modseq: r.modseq, sizeBytes: r.size_bytes,
|
|
@@ -528,7 +633,7 @@ function create(opts) {
|
|
|
528
633
|
},
|
|
529
634
|
setFlags: function (folderName, objectids, flagOpts) {
|
|
530
635
|
return _setFlags({
|
|
531
|
-
db: db,
|
|
636
|
+
db: db, messagesTable: messagesTable,
|
|
532
637
|
stmtGetFolderByName: stmtGetFolderByName,
|
|
533
638
|
stmtBumpFolderModseq: stmtBumpFolderModseq,
|
|
534
639
|
stmtSetFlag: stmtSetFlag,
|
|
@@ -549,12 +654,13 @@ function create(opts) {
|
|
|
549
654
|
var role = fo.role || null;
|
|
550
655
|
var parentId = fo.parentId || null;
|
|
551
656
|
var uidvalidity = Math.floor(Date.now() / 1000); // Unix timestamp, not bytes
|
|
552
|
-
stmtInsertFolder.run(name, role, parentId, uidvalidity);
|
|
657
|
+
stmtInsertFolder.run(name, role, parentId, 0, uidvalidity); // modseq_max seeds to 0
|
|
553
658
|
return stmtGetFolderByName.get(name);
|
|
554
659
|
},
|
|
555
660
|
listFolders: function () { return stmtListFolders.all(); },
|
|
556
661
|
threadFor: function (objectid) {
|
|
557
|
-
var msg = db.prepare(
|
|
662
|
+
var msg = db.prepare(sql.select(messagesTable, SQL)
|
|
663
|
+
.columns(["thread_root_id"]).where("objectid", "?").toSql().sql).get(objectid);
|
|
558
664
|
if (!msg) return [];
|
|
559
665
|
return stmtThreadFor.all(msg.thread_root_id).map(function (r) { return r.objectid; });
|
|
560
666
|
},
|
|
@@ -798,10 +904,10 @@ function _appendMessage(args) {
|
|
|
798
904
|
sealed.received_at, sealed.size_bytes, sealed.message_id, sealed.message_id_hash,
|
|
799
905
|
sealed.in_reply_to, sealed.references_csv, sealed.thread_root_id,
|
|
800
906
|
sealed.subject, sealed.from_addr, sealed.from_hash, sealed.to_addrs,
|
|
801
|
-
sealed.body_text, sealed.body_html
|
|
907
|
+
sealed.body_text, sealed.body_html, 0 // legal_hold seeds to 0
|
|
802
908
|
);
|
|
803
909
|
args.stmtBumpFolderModseq.run(modseq, args.folderName);
|
|
804
|
-
args.stmtBumpQuota.run(folder.id, buf.length, 1);
|
|
910
|
+
args.stmtBumpQuota.run(folder.id, buf.length, 1, null, null); // cap_bytes / cap_count seed NULL on first insert
|
|
805
911
|
|
|
806
912
|
// FTS index update — tokenize the PRE-seal plaintext, hash each
|
|
807
913
|
// token with the per-deployment vault salt, insert into the FTS5
|
|
@@ -896,7 +1002,7 @@ function _moveMessages(args) {
|
|
|
896
1002
|
// on append; move must keep both sides accurate.
|
|
897
1003
|
if (changed > 0) {
|
|
898
1004
|
args.stmtDecrementQuota.run(movedBytes, changed, fromFolder.id);
|
|
899
|
-
args.stmtBumpQuota.run(toFolder.id, movedBytes, changed);
|
|
1005
|
+
args.stmtBumpQuota.run(toFolder.id, movedBytes, changed, null, null); // cap_bytes / cap_count seed NULL on first insert
|
|
900
1006
|
}
|
|
901
1007
|
args.stmtBumpFolderModseq.run(srcModseq, args.fromFolderName);
|
|
902
1008
|
args.stmtBumpFolderModseq.run(dstModseq, args.toFolderName);
|
|
@@ -941,9 +1047,15 @@ function _setFlags(args) {
|
|
|
941
1047
|
var CHUNK = 500; // IN-clause chunk size, not bytes
|
|
942
1048
|
for (var i = 0; i < args.objectids.length; i += CHUNK) {
|
|
943
1049
|
var chunk = args.objectids.slice(i, i + CHUNK);
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
1050
|
+
// b.sql.update(...).whereIn(objectid, chunk) emits the per-chunk
|
|
1051
|
+
// `SET "modseq" = ? WHERE "objectid" IN (?, ?, ...)` text (the chunk
|
|
1052
|
+
// entries are `?` placeholders, bound positionally at run()); the
|
|
1053
|
+
// modseq `?` leads, so the bind list is [newModseq].concat(chunk).
|
|
1054
|
+
var stmtText = sql.update(args.messagesTable, { dialect: "sqlite", quoteName: true })
|
|
1055
|
+
.set("modseq", "?")
|
|
1056
|
+
.whereIn("objectid", chunk.map(function () { return "?"; }))
|
|
1057
|
+
.toSql().sql;
|
|
1058
|
+
var stmt = args.db.prepare(stmtText);
|
|
947
1059
|
stmt.run.apply(stmt, [newModseq].concat(chunk));
|
|
948
1060
|
}
|
|
949
1061
|
}
|
|
@@ -1009,100 +1121,121 @@ function _normalizeMsgId(s) {
|
|
|
1009
1121
|
|
|
1010
1122
|
// ---- Schema bootstrap ----------------------------------------------------
|
|
1011
1123
|
|
|
1012
|
-
function _ensureSchema(db,
|
|
1124
|
+
function _ensureSchema(db, tables) {
|
|
1125
|
+
// Every DDL statement is composed through b.sql with quoteName so the
|
|
1126
|
+
// prefixed table name emits as a quoted identifier against the concrete
|
|
1127
|
+
// sqlite handle (no clusterStorage rewrite — the store owns its backend).
|
|
1128
|
+
// The DDL builders return { sql } (DDL binds no values).
|
|
1129
|
+
var DDL = { dialect: "sqlite", quoteName: true };
|
|
1130
|
+
var foldersTable = tables.foldersTable;
|
|
1131
|
+
var messagesTable = tables.messagesTable;
|
|
1132
|
+
var flagsTable = tables.flagsTable;
|
|
1133
|
+
var quotaTable = tables.quotaTable;
|
|
1134
|
+
var ftsTable = tables.ftsTable;
|
|
1135
|
+
var metaTable = tables.metaTable;
|
|
1136
|
+
|
|
1137
|
+
function _ddl(built) { db.prepare(built.sql).run(); }
|
|
1138
|
+
|
|
1013
1139
|
// Folders table — created first since messages reference folder_id.
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
"
|
|
1019
|
-
"
|
|
1020
|
-
|
|
1021
|
-
"
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
//
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
"modseq
|
|
1034
|
-
"internal_date
|
|
1035
|
-
"received_at
|
|
1036
|
-
"size_bytes
|
|
1037
|
-
"message_id
|
|
1038
|
-
"message_id_hash
|
|
1039
|
-
"in_reply_to
|
|
1040
|
-
"references_csv
|
|
1041
|
-
"thread_root_id
|
|
1042
|
-
"subject
|
|
1043
|
-
"from_addr
|
|
1044
|
-
"from_hash
|
|
1045
|
-
"to_addrs
|
|
1046
|
-
"body_text
|
|
1047
|
-
"body_html
|
|
1048
|
-
"legal_hold
|
|
1049
|
-
|
|
1050
|
-
).run();
|
|
1140
|
+
// `id` is an auto-increment PK (sqlite emits INTEGER PRIMARY KEY
|
|
1141
|
+
// AUTOINCREMENT, the rowid-backed identity column).
|
|
1142
|
+
_ddl(sql.createTable(foldersTable, [
|
|
1143
|
+
{ name: "id", autoIncrement: true },
|
|
1144
|
+
{ name: "name", type: "text", notNull: true, unique: true },
|
|
1145
|
+
{ name: "role", type: "text" },
|
|
1146
|
+
{ name: "parent_id", type: "int" },
|
|
1147
|
+
{ name: "modseq_max", type: "int", notNull: true, default: 0 },
|
|
1148
|
+
{ name: "uidvalidity", type: "int", notNull: true },
|
|
1149
|
+
], DDL));
|
|
1150
|
+
_ddl(sql.createIndex(foldersTable + "_role_idx", foldersTable, ["role"], DDL));
|
|
1151
|
+
|
|
1152
|
+
// Messages table — sealed-by-default subject / from / to / body. The
|
|
1153
|
+
// folder_id FK inherits the quoteName so the referenced table resolves
|
|
1154
|
+
// to the same concrete identifier.
|
|
1155
|
+
_ddl(sql.createTable(messagesTable, [
|
|
1156
|
+
{ name: "objectid", type: "text", primaryKey: true },
|
|
1157
|
+
{ name: "folder_id", type: "int", notNull: true,
|
|
1158
|
+
references: { table: foldersTable, column: "id" } },
|
|
1159
|
+
{ name: "modseq", type: "int", notNull: true },
|
|
1160
|
+
{ name: "internal_date", type: "int", notNull: true },
|
|
1161
|
+
{ name: "received_at", type: "int", notNull: true },
|
|
1162
|
+
{ name: "size_bytes", type: "int", notNull: true },
|
|
1163
|
+
{ name: "message_id", type: "text" },
|
|
1164
|
+
{ name: "message_id_hash", type: "text" },
|
|
1165
|
+
{ name: "in_reply_to", type: "text" },
|
|
1166
|
+
{ name: "references_csv", type: "text" },
|
|
1167
|
+
{ name: "thread_root_id", type: "text", notNull: true },
|
|
1168
|
+
{ name: "subject", type: "text" },
|
|
1169
|
+
{ name: "from_addr", type: "text" },
|
|
1170
|
+
{ name: "from_hash", type: "text" },
|
|
1171
|
+
{ name: "to_addrs", type: "text" },
|
|
1172
|
+
{ name: "body_text", type: "text" },
|
|
1173
|
+
{ name: "body_html", type: "text" },
|
|
1174
|
+
{ name: "legal_hold", type: "int", notNull: true, default: 0 },
|
|
1175
|
+
], DDL));
|
|
1051
1176
|
// Indexes — modseq for CONDSTORE, thread_root_id for thread fetch,
|
|
1052
1177
|
// message_id_hash for threading lookup, from_hash for sender search.
|
|
1053
1178
|
["modseq", "thread_root_id", "message_id_hash", "from_hash", "received_at", "legal_hold"]
|
|
1054
1179
|
.forEach(function (col) {
|
|
1055
|
-
|
|
1056
|
-
"CREATE INDEX IF NOT EXISTS " + safeSql.quoteIdentifier(qMsgs.slice(1, -1) + "_" + col + "_idx", "sqlite") +
|
|
1057
|
-
" ON " + qMsgs + "(" + safeSql.quoteIdentifier(col, "sqlite") + ")"
|
|
1058
|
-
).run();
|
|
1180
|
+
_ddl(sql.createIndex(messagesTable + "_" + col + "_idx", messagesTable, [col], DDL));
|
|
1059
1181
|
});
|
|
1060
1182
|
|
|
1061
|
-
// Flags table — many-to-one with messages.
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
"objectid
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
).run();
|
|
1183
|
+
// Flags table — many-to-one with messages. Composite (objectid, flag) PK
|
|
1184
|
+
// + ON DELETE CASCADE FK back to the messages table.
|
|
1185
|
+
_ddl(sql.createTable(flagsTable, [
|
|
1186
|
+
{ name: "objectid", type: "text", notNull: true,
|
|
1187
|
+
references: { table: messagesTable, column: "objectid", onDelete: "CASCADE" } },
|
|
1188
|
+
{ name: "flag", type: "text", notNull: true },
|
|
1189
|
+
{ name: "set_at", type: "int", notNull: true },
|
|
1190
|
+
], Object.assign({ primaryKey: ["objectid", "flag"] }, DDL)));
|
|
1070
1191
|
|
|
1071
1192
|
// Quota table — per-folder counters bumped atomically with append/delete.
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
"used_bytes
|
|
1076
|
-
"used_count
|
|
1077
|
-
"cap_bytes
|
|
1078
|
-
"cap_count
|
|
1079
|
-
|
|
1080
|
-
).run();
|
|
1193
|
+
_ddl(sql.createTable(quotaTable, [
|
|
1194
|
+
{ name: "folder_id", type: "int", primaryKey: true,
|
|
1195
|
+
references: { table: foldersTable, column: "id" } },
|
|
1196
|
+
{ name: "used_bytes", type: "int", notNull: true, default: 0 },
|
|
1197
|
+
{ name: "used_count", type: "int", notNull: true, default: 0 },
|
|
1198
|
+
{ name: "cap_bytes", type: "int" },
|
|
1199
|
+
{ name: "cap_count", type: "int" },
|
|
1200
|
+
], DDL));
|
|
1081
1201
|
|
|
1082
1202
|
// Sealed-token FTS5 virtual table. The token-hash transform lives in
|
|
1083
1203
|
// `lib/mail-store-fts.js`; this is the storage layer. Tokenizer is
|
|
1084
1204
|
// `unicode61 remove_diacritics 2` so FTS5's segmenter splits hash-
|
|
1085
1205
|
// tokens on whitespace exactly — hashes are ASCII-hex-only, so no
|
|
1086
|
-
// Unicode case-fold runs at MATCH time.
|
|
1087
|
-
|
|
1206
|
+
// Unicode case-fold runs at MATCH time. objectid is UNINDEXED (the join
|
|
1207
|
+
// key, stored but not searched).
|
|
1208
|
+
_ddl(sql.createVirtualTable(ftsTable, {
|
|
1209
|
+
columns: [
|
|
1210
|
+
{ name: "objectid", unindexed: true },
|
|
1211
|
+
"subject_toks", "addr_toks", "body_toks",
|
|
1212
|
+
],
|
|
1213
|
+
tokenize: "unicode61 remove_diacritics 2",
|
|
1214
|
+
}));
|
|
1088
1215
|
|
|
1089
1216
|
// Per-prefix key/value metadata table. Holds the FTS on-disk format
|
|
1090
1217
|
// marker (`fts_format`) so the reindex path can detect a stale index
|
|
1091
1218
|
// and rebuild it. Scoped per table-prefix (NOT PRAGMA user_version,
|
|
1092
1219
|
// which is db-global and would collide across stores sharing one
|
|
1093
1220
|
// sqlite file).
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
).run();
|
|
1221
|
+
_ddl(sql.createTable(metaTable, [
|
|
1222
|
+
{ name: "key", type: "text", primaryKey: true },
|
|
1223
|
+
{ name: "value", type: "text" },
|
|
1224
|
+
], DDL));
|
|
1099
1225
|
}
|
|
1100
1226
|
|
|
1101
|
-
function _ensureDefaultFolders(db,
|
|
1102
|
-
|
|
1103
|
-
|
|
1227
|
+
function _ensureDefaultFolders(db, foldersTable) {
|
|
1228
|
+
// INSERT OR IGNORE — composed as ON CONFLICT(name) DO NOTHING (the
|
|
1229
|
+
// folders UNIQUE name) so re-bootstrapping an existing store is a no-op.
|
|
1230
|
+
// Every column is a `?` placeholder bound positionally at run() (the
|
|
1231
|
+
// builder emits one `?` per value; parent_id binds NULL, modseq_max 0).
|
|
1232
|
+
var stmtText = sql.upsert(foldersTable, { dialect: "sqlite", quoteName: true })
|
|
1233
|
+
.columns(["name", "role", "parent_id", "modseq_max", "uidvalidity"])
|
|
1234
|
+
.values({ name: "?", role: "?", parent_id: "?", modseq_max: "?", uidvalidity: "?" })
|
|
1235
|
+
.onConflict(["name"]).doNothing().toSql().sql;
|
|
1236
|
+
var stmt = db.prepare(stmtText);
|
|
1104
1237
|
var uv = Math.floor(Date.now() / 1000); // Unix timestamp, not bytes
|
|
1105
|
-
DEFAULT_FOLDERS.forEach(function (f) { stmt.run(f.name, f.role, uv); });
|
|
1238
|
+
DEFAULT_FOLDERS.forEach(function (f) { stmt.run(f.name, f.role, null, 0, uv); });
|
|
1106
1239
|
}
|
|
1107
1240
|
|
|
1108
1241
|
module.exports = {
|