@rapidmx/web-client 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/admin/branding/index.tsx +39 -404
- package/apps/admin/domains/[uid].tsx +92 -259
- package/apps/admin/encryption-policy/index.tsx +19 -0
- package/apps/admin/index.tsx +100 -82
- package/apps/admin/mailbox-policy/index.tsx +19 -0
- package/apps/admin/mailboxes/new/index.tsx +28 -291
- package/apps/admin/plugins/index.tsx +15 -0
- package/apps/admin/retention-policy/index.tsx +39 -132
- package/apps/admin/setup/index.tsx +15 -0
- package/apps/shared/components/admin/layout/AdminShell.tsx +249 -210
- package/apps/shared/components/admin/settings/BrandingForm.tsx +374 -0
- package/apps/shared/components/admin/settings/DomainDnsSetup.tsx +176 -0
- package/apps/shared/components/admin/settings/EncryptionPolicyForm.tsx +104 -0
- package/apps/shared/components/admin/settings/LoadedSettingsForm.tsx +34 -0
- package/apps/shared/components/admin/settings/MailboxCreateForm.tsx +302 -0
- package/apps/shared/components/admin/settings/MailboxPolicyForm.tsx +119 -0
- package/apps/shared/components/admin/settings/PluginsManager.tsx +595 -0
- package/apps/shared/components/admin/settings/RetentionPolicyForm.tsx +98 -0
- package/apps/shared/components/admin/setup/EscrowSetupStep.tsx +254 -0
- package/apps/shared/components/admin/setup/SetupWizard.tsx +300 -0
- package/apps/shared/components/calendar/CalendarListSidebar.tsx +120 -76
- package/apps/shared/components/calendar/EventModal.tsx +40 -4
- package/apps/shared/components/calendar/layout/CalendarShell.tsx +80 -99
- package/apps/shared/components/contacts/ContactForm.tsx +38 -4
- package/apps/shared/components/layout/AppShell.tsx +198 -169
- package/apps/shared/components/layout/KeyEnrollmentGate.tsx +298 -265
- package/apps/shared/components/layout/UnlockPromptProvider.tsx +128 -0
- package/apps/shared/components/mail/MessageDetailPane.tsx +630 -595
- package/apps/shared/components/mail/compose/ComposeContext.tsx +8 -3
- package/apps/shared/components/mail/compose/ComposeWindow.tsx +930 -765
- package/apps/shared/components/mail/layout/MailShell.tsx +408 -302
- package/apps/shared/components/settings/layout/SettingsShell.tsx +1 -0
- package/apps/shared/mail/findWellKnownFolderUid.ts +13 -0
- package/apps/shared/search/LocalIndexLifecycle.tsx +89 -0
- package/apps/shared/search/localIndexBlockCipher.ts +83 -0
- package/apps/shared/search/localIndexBuilder.ts +179 -0
- package/apps/shared/search/localIndexKey.ts +27 -0
- package/apps/shared/search/localIndexRpcClient.ts +116 -0
- package/apps/shared/search/localIndexSchema.ts +227 -0
- package/apps/shared/search/localIndexSizePreference.ts +88 -0
- package/apps/shared/search/localIndexVFS.ts +235 -0
- package/apps/shared/search/localIndexWorker.ts +464 -0
- package/apps/shared/search/searchTier2.ts +79 -0
- package/apps/shared/search/wa-sqlite-shims.d.ts +44 -0
- package/apps/www/calendar/index.tsx +31 -21
- package/apps/www/contacts/index.tsx +33 -6
- package/apps/www/index.tsx +661 -106
- package/apps/www/messages/[uid].tsx +5 -1
- package/apps/www/settings/encryption/index.tsx +63 -1
- package/apps/www/settings/sharing/index.tsx +271 -0
- package/apps/www/tasks/index.tsx +53 -4
- package/dist/apps/admin/branding/index.js +4 -98
- package/dist/apps/admin/domains/[uid].js +5 -68
- package/dist/apps/admin/encryption-policy/index.js +8 -0
- package/dist/apps/admin/index.js +14 -1
- package/dist/apps/admin/mailbox-policy/index.js +8 -0
- package/dist/apps/admin/mailboxes/new/index.js +4 -89
- package/dist/apps/admin/plugins/index.js +6 -0
- package/dist/apps/admin/retention-policy/index.js +3 -36
- package/dist/apps/admin/setup/index.js +6 -0
- package/dist/apps/shared/components/admin/layout/AdminShell.js +34 -3
- package/dist/apps/shared/components/admin/settings/BrandingForm.js +105 -0
- package/dist/apps/shared/components/admin/settings/DomainDnsSetup.js +77 -0
- package/dist/apps/shared/components/admin/settings/EncryptionPolicyForm.js +57 -0
- package/dist/apps/shared/components/admin/settings/LoadedSettingsForm.js +25 -0
- package/dist/apps/shared/components/admin/settings/MailboxCreateForm.js +106 -0
- package/dist/apps/shared/components/admin/settings/MailboxPolicyForm.js +52 -0
- package/dist/apps/shared/components/admin/settings/PluginsManager.js +260 -0
- package/dist/apps/shared/components/admin/settings/RetentionPolicyForm.js +44 -0
- package/dist/apps/shared/components/admin/setup/EscrowSetupStep.js +131 -0
- package/dist/apps/shared/components/admin/setup/SetupWizard.js +142 -0
- package/dist/apps/shared/components/calendar/CalendarListSidebar.js +24 -13
- package/dist/apps/shared/components/calendar/EventModal.js +16 -4
- package/dist/apps/shared/components/calendar/layout/CalendarShell.js +47 -39
- package/dist/apps/shared/components/contacts/ContactForm.js +16 -5
- package/dist/apps/shared/components/layout/AppShell.js +30 -7
- package/dist/apps/shared/components/layout/KeyEnrollmentGate.js +15 -3
- package/dist/apps/shared/components/layout/UnlockPromptProvider.js +77 -0
- package/dist/apps/shared/components/mail/MessageDetailPane.js +27 -2
- package/dist/apps/shared/components/mail/compose/ComposeContext.js +2 -2
- package/dist/apps/shared/components/mail/compose/ComposeWindow.js +134 -14
- package/dist/apps/shared/components/mail/layout/MailShell.js +106 -40
- package/dist/apps/shared/components/settings/layout/SettingsShell.js +1 -0
- package/dist/apps/shared/mail/findWellKnownFolderUid.js +12 -0
- package/dist/apps/shared/search/LocalIndexLifecycle.js +76 -0
- package/dist/apps/shared/search/localIndexBlockCipher.js +63 -0
- package/dist/apps/shared/search/localIndexBuilder.js +153 -0
- package/dist/apps/shared/search/localIndexKey.js +25 -0
- package/dist/apps/shared/search/localIndexRpcClient.js +78 -0
- package/dist/apps/shared/search/localIndexSchema.js +179 -0
- package/dist/apps/shared/search/localIndexSizePreference.js +78 -0
- package/dist/apps/shared/search/localIndexVFS.js +230 -0
- package/dist/apps/shared/search/localIndexWorker.js +341 -0
- package/dist/apps/shared/search/searchTier2.js +38 -0
- package/dist/apps/www/calendar/index.js +14 -12
- package/dist/apps/www/contacts/index.js +15 -6
- package/dist/apps/www/index.js +496 -94
- package/dist/apps/www/messages/[uid].js +5 -1
- package/dist/apps/www/settings/encryption/index.js +30 -2
- package/dist/apps/www/settings/sharing/index.js +128 -0
- package/dist/apps/www/tasks/index.js +27 -5
- package/package.json +3 -2
|
@@ -0,0 +1,464 @@
|
|
|
1
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// Copyright (C) 2026 Jean-Philippe Steinmetz
|
|
3
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
5
|
+
/**
|
|
6
|
+
* The Tier 2 local search index's Worker entry point (`specs/search.md` §13) - runs a WASM SQLite build
|
|
7
|
+
* (`@journeyapps/wa-sqlite`) over `EncryptingVFS` (`localIndexVFS.ts`), an OPFS-backed, AES-256-GCM
|
|
8
|
+
* page-encrypting VFS. Must live in a Worker: OPFS synchronous access handles (what the inner
|
|
9
|
+
* `AccessHandlePoolVFS` - the "SAH Pool VFS" the spec names - uses) are only available off the UI
|
|
10
|
+
* thread, which also satisfies the spec's "indexing MUST NOT run on the UI thread" requirement for free.
|
|
11
|
+
*
|
|
12
|
+
* Boots the **Asyncify** build (`dist/wa-sqlite-async.mjs`), not the plain synchronous one - required
|
|
13
|
+
* because `EncryptingVFS`'s `jRead`/`jWrite` are genuinely async (WebCrypto has no synchronous form) -
|
|
14
|
+
* see that file's own doc comment for why this doesn't conflict with `AccessHandlePoolVFS` itself being
|
|
15
|
+
* synchronous underneath it.
|
|
16
|
+
*
|
|
17
|
+
* One SQLite connection per mailbox, opened on `init` and kept for the Worker's lifetime (or until
|
|
18
|
+
* `destroy`). `entity_uid` is this module's identifier for a message (the only entity type Tier 2 covers
|
|
19
|
+
* today - see the doc comment on `searchTier3.ts`'s identical scope decision, which this mirrors).
|
|
20
|
+
*
|
|
21
|
+
* Real index/search/lifecycle RPC methods are all implemented below; `selfTest` stays alongside them as
|
|
22
|
+
* an internal diagnostic (proves the encrypted round trip end to end: write through `EncryptingVFS`,
|
|
23
|
+
* close the connection, reopen, read back) rather than being removed once the real surface existed.
|
|
24
|
+
*/
|
|
25
|
+
import SQLiteESMFactory from "@journeyapps/wa-sqlite/dist/wa-sqlite-async.mjs";
|
|
26
|
+
import * as SQLite from "@journeyapps/wa-sqlite";
|
|
27
|
+
import type { ParsedSearchQuery } from "@rapidmx/react-shared/search/queryGrammar.js";
|
|
28
|
+
import { EncryptingVFS } from "./localIndexVFS.js";
|
|
29
|
+
import {
|
|
30
|
+
BM25_WEIGHTS_SQL,
|
|
31
|
+
CREATE_SCHEMA_SQL,
|
|
32
|
+
SCHEMA_VERSION,
|
|
33
|
+
UPSERT_ENTITY_SQL,
|
|
34
|
+
LocalIndexEntity,
|
|
35
|
+
buildMatchExpression,
|
|
36
|
+
buildSearchPredicates,
|
|
37
|
+
entityBindValues,
|
|
38
|
+
} from "./localIndexSchema.js";
|
|
39
|
+
|
|
40
|
+
/** One request sent from the main thread. `id` correlates the eventual response - `postMessage` has no
|
|
41
|
+
* native request/response pairing, so every RPC call carries its own id (see `localIndexRpcClient.ts`,
|
|
42
|
+
* the main-thread counterpart that generates/awaits these). */
|
|
43
|
+
export interface LocalIndexRequest {
|
|
44
|
+
id: number;
|
|
45
|
+
method: "init" | "indexEntities" | "removeEntity" | "search" | "coverage" | "setWindow" | "setBuilding" | "destroy" | "selfTest" | "ping";
|
|
46
|
+
params?: unknown;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export type LocalIndexResponse =
|
|
50
|
+
| { id: number; ok: true; result: unknown }
|
|
51
|
+
| { id: number; ok: false; error: string };
|
|
52
|
+
|
|
53
|
+
export interface InitParams {
|
|
54
|
+
mailboxUid: string;
|
|
55
|
+
/** The mailbox's already-derived local-index key (`localIndexKey.ts`'s `deriveLocalIndexKey()`) -
|
|
56
|
+
* this Worker never touches the master key itself, only this one purpose-derived value, matching
|
|
57
|
+
* every other MK-derived-key boundary already established elsewhere in this codebase. */
|
|
58
|
+
indexKey: Uint8Array;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface IndexEntitiesParams {
|
|
62
|
+
mailboxUid: string;
|
|
63
|
+
entities: LocalIndexEntity[];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface RemoveEntityParams {
|
|
67
|
+
mailboxUid: string;
|
|
68
|
+
entityUid: string;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface SearchParams {
|
|
72
|
+
mailboxUid: string;
|
|
73
|
+
parsed: ParsedSearchQuery;
|
|
74
|
+
limit: number;
|
|
75
|
+
/** How many matches to skip before returning `limit` more - §8's composite pagination cursor's own
|
|
76
|
+
* "the local index position consumed so far" component, threaded straight through to SQL `OFFSET`.
|
|
77
|
+
* Defaults to `0` for a first page. */
|
|
78
|
+
offset?: number;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export interface LocalSearchHit {
|
|
82
|
+
entityUid: string;
|
|
83
|
+
/** Raw `bm25()` score - more negative is a better match, per SQLite's own convention. Normalized by
|
|
84
|
+
* `searchTier2.ts` (main thread) via `searchScoring.ts`'s shared `normalizeServerScores()`, the same
|
|
85
|
+
* way every other tier's raw score is, before merging (spec §7). */
|
|
86
|
+
score: number;
|
|
87
|
+
snippet?: string;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface LocalSearchPage {
|
|
91
|
+
hits: LocalSearchHit[];
|
|
92
|
+
/** `true` when at least one more match exists beyond this page - detected by fetching `limit + 1`
|
|
93
|
+
* rows and trimming the extra one, rather than a separate `COUNT(*)` query. */
|
|
94
|
+
hasMore: boolean;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export interface Coverage {
|
|
98
|
+
/** Oldest `date_for_sort` currently covered, or `undefined` for an empty index. */
|
|
99
|
+
indexedFrom?: string;
|
|
100
|
+
indexedCount: number;
|
|
101
|
+
building: boolean;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export interface SetWindowParams {
|
|
105
|
+
mailboxUid: string;
|
|
106
|
+
timeFloorMonths: number;
|
|
107
|
+
byteBudgetBytes: number;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export interface SetBuildingParams {
|
|
111
|
+
mailboxUid: string;
|
|
112
|
+
building: boolean;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
interface OpenConnection {
|
|
116
|
+
sqlite3: SQLiteAPI;
|
|
117
|
+
db: number;
|
|
118
|
+
vfs: EncryptingVFS;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Keyed by `mailboxUid` - a Worker instance is per-tab, not per-mailbox, so this stays a map even
|
|
122
|
+
* though only one mailbox is ever unlocked in this app's UI at a time today. */
|
|
123
|
+
const connections = new Map<string, OpenConnection>();
|
|
124
|
+
|
|
125
|
+
/** The OPFS directory name (and `EncryptingVFS` name) a mailbox's index lives under - scoped per
|
|
126
|
+
* mailbox so two mailboxes' indexes never collide and `destroy(mailboxUid)` (added in the next pass) can
|
|
127
|
+
* remove exactly one without touching the others. */
|
|
128
|
+
function poolNameFor(mailboxUid: string): string {
|
|
129
|
+
return `rapidmx-localsearch-${mailboxUid}`;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
async function openConnection({ mailboxUid, indexKey }: InitParams): Promise<OpenConnection> {
|
|
133
|
+
const module = await SQLiteESMFactory();
|
|
134
|
+
const sqlite3 = SQLite.Factory(module);
|
|
135
|
+
const vfs = await EncryptingVFS.create(poolNameFor(mailboxUid), module, indexKey);
|
|
136
|
+
sqlite3.vfs_register(vfs, true);
|
|
137
|
+
const db = await sqlite3.open_v2("index.db");
|
|
138
|
+
// No WAL, no rollback journal - see localIndexVFS.ts's own doc comment on why this index's lack of
|
|
139
|
+
// a durability requirement makes that an acceptable, deliberate simplification here.
|
|
140
|
+
await sqlite3.exec(db, "PRAGMA journal_mode=OFF; PRAGMA page_size=4096;");
|
|
141
|
+
await sqlite3.exec(db, CREATE_SCHEMA_SQL);
|
|
142
|
+
const connection = { sqlite3, db, vfs };
|
|
143
|
+
|
|
144
|
+
const storedVersion = await readMeta(connection, "schema_version");
|
|
145
|
+
if (storedVersion !== String(SCHEMA_VERSION)) {
|
|
146
|
+
// §11 "Invalidation... discarded and rebuilt... on schema version change" - drop every table's
|
|
147
|
+
// rows (the DDL itself is `CREATE ... IF NOT EXISTS`, already current) and start fresh, rather
|
|
148
|
+
// than attempting to migrate content built under an incompatible schema.
|
|
149
|
+
await sqlite3.exec(connection.db, "DELETE FROM entities; DELETE FROM entities_fts;");
|
|
150
|
+
await writeMeta(connection, "schema_version", String(SCHEMA_VERSION));
|
|
151
|
+
}
|
|
152
|
+
return connection;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async function readMeta(connection: OpenConnection, key: string): Promise<string | undefined> {
|
|
156
|
+
let value: string | undefined;
|
|
157
|
+
for await (const stmt of connection.sqlite3.statements(connection.db, "SELECT value FROM meta WHERE key = ?")) {
|
|
158
|
+
connection.sqlite3.bind_collection(stmt, [key]);
|
|
159
|
+
if ((await connection.sqlite3.step(stmt)) === SQLite.SQLITE_ROW) {
|
|
160
|
+
value = connection.sqlite3.column(stmt, 0) as string;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return value;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async function writeMeta(connection: OpenConnection, key: string, value: string): Promise<void> {
|
|
167
|
+
for await (const stmt of connection.sqlite3.statements(connection.db, "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")) {
|
|
168
|
+
connection.sqlite3.bind_collection(stmt, [key, value]);
|
|
169
|
+
await connection.sqlite3.step(stmt);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async function init(params: InitParams): Promise<void> {
|
|
174
|
+
if (connections.has(params.mailboxUid)) {
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
connections.set(params.mailboxUid, await openConnection(params));
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function requireConnection(mailboxUid: string): OpenConnection {
|
|
181
|
+
const connection = connections.get(mailboxUid);
|
|
182
|
+
if (!connection) {
|
|
183
|
+
throw new Error(`localIndexWorker: init() was never called for mailbox ${mailboxUid}`);
|
|
184
|
+
}
|
|
185
|
+
return connection;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** Tears down a mailbox's connection completely: the SQLite connection itself (`sqlite3.close(db)`) AND
|
|
189
|
+
* the underlying `EncryptingVFS`/`AccessHandlePoolVFS` instance (`vfs.close()`) - two separate lifecycles
|
|
190
|
+
* (see `EncryptingVFS.close()`'s own doc comment on why skipping the second one breaks re-`init()`ing the
|
|
191
|
+
* same mailbox). Removes the entry from `connections` either way. */
|
|
192
|
+
async function closeConnection(mailboxUid: string): Promise<void> {
|
|
193
|
+
const connection = connections.get(mailboxUid);
|
|
194
|
+
if (!connection) {
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
connections.delete(mailboxUid);
|
|
198
|
+
await connection.sqlite3.close(connection.db);
|
|
199
|
+
await connection.vfs.close();
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
async function sumBytes(connection: OpenConnection): Promise<number> {
|
|
203
|
+
let total = 0;
|
|
204
|
+
for await (const stmt of connection.sqlite3.statements(connection.db, "SELECT COALESCE(SUM(byte_size), 0) FROM entities")) {
|
|
205
|
+
if ((await connection.sqlite3.step(stmt)) === SQLite.SQLITE_ROW) {
|
|
206
|
+
total = connection.sqlite3.column(stmt, 0) as number;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return total;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
async function oldestDateForSort(connection: OpenConnection): Promise<string | undefined> {
|
|
213
|
+
let oldest: string | undefined;
|
|
214
|
+
for await (const stmt of connection.sqlite3.statements(connection.db, "SELECT MIN(date_for_sort) FROM entities")) {
|
|
215
|
+
if ((await connection.sqlite3.step(stmt)) === SQLite.SQLITE_ROW) {
|
|
216
|
+
oldest = (connection.sqlite3.column(stmt, 0) as string | null) ?? undefined;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return oldest;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async function entityCount(connection: OpenConnection): Promise<number> {
|
|
223
|
+
let count = 0;
|
|
224
|
+
for await (const stmt of connection.sqlite3.statements(connection.db, "SELECT COUNT(*) FROM entities")) {
|
|
225
|
+
if ((await connection.sqlite3.step(stmt)) === SQLite.SQLITE_ROW) {
|
|
226
|
+
count = connection.sqlite3.column(stmt, 0) as number;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return count;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Deletes the single oldest entity (by `date_for_sort`) and returns whether one existed to delete -
|
|
233
|
+
* `entities_ad` (see `localIndexSchema.ts`) keeps `entities_fts` in sync automatically. One row per call
|
|
234
|
+
* (not a batch `DELETE ... LIMIT`, which SQLite's default build doesn't compile in) so
|
|
235
|
+
* `#applyEviction()`'s own loop can re-check the byte total after each deletion rather than
|
|
236
|
+
* over-evicting. */
|
|
237
|
+
async function deleteOldestEntity(connection: OpenConnection): Promise<boolean> {
|
|
238
|
+
let deleted = false;
|
|
239
|
+
for await (const stmt of connection.sqlite3.statements(
|
|
240
|
+
connection.db,
|
|
241
|
+
"DELETE FROM entities WHERE rowid = (SELECT rowid FROM entities ORDER BY date_for_sort ASC LIMIT 1)",
|
|
242
|
+
)) {
|
|
243
|
+
await connection.sqlite3.step(stmt);
|
|
244
|
+
deleted = connection.sqlite3.changes(connection.db) > 0;
|
|
245
|
+
}
|
|
246
|
+
return deleted;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** Oldest-first eviction against the configured byte budget (spec §11 "Eviction... MUST NOT block
|
|
250
|
+
* search" - this runs to completion as part of `indexEntities()`, which is already off the UI thread by
|
|
251
|
+
* virtue of running in this Worker, so there's no separate scheduling concern here). A no-op when no
|
|
252
|
+
* budget has been configured yet (`setWindow()` was never called) - nothing to enforce. */
|
|
253
|
+
async function applyEviction(connection: OpenConnection): Promise<void> {
|
|
254
|
+
const byteBudgetRaw = await readMeta(connection, "byte_budget");
|
|
255
|
+
const byteBudget = byteBudgetRaw ? Number(byteBudgetRaw) : undefined;
|
|
256
|
+
if (!byteBudget) {
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
for (;;) {
|
|
260
|
+
const total = await sumBytes(connection);
|
|
261
|
+
if (total <= byteBudget) {
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
const deletedOne = await deleteOldestEntity(connection);
|
|
265
|
+
if (!deletedOne) {
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
async function indexEntities({ mailboxUid, entities }: IndexEntitiesParams): Promise<void> {
|
|
272
|
+
const connection = requireConnection(mailboxUid);
|
|
273
|
+
for (const entity of entities) {
|
|
274
|
+
for await (const stmt of connection.sqlite3.statements(connection.db, UPSERT_ENTITY_SQL)) {
|
|
275
|
+
connection.sqlite3.bind_collection(stmt, entityBindValues(entity));
|
|
276
|
+
await connection.sqlite3.step(stmt);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
await applyEviction(connection);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
async function removeEntity({ mailboxUid, entityUid }: RemoveEntityParams): Promise<void> {
|
|
283
|
+
const connection = requireConnection(mailboxUid);
|
|
284
|
+
for await (const stmt of connection.sqlite3.statements(connection.db, "DELETE FROM entities WHERE entity_uid = ?")) {
|
|
285
|
+
connection.sqlite3.bind_collection(stmt, [entityUid]);
|
|
286
|
+
await connection.sqlite3.step(stmt);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
async function search({ mailboxUid, parsed, limit, offset = 0 }: SearchParams): Promise<LocalSearchPage> {
|
|
291
|
+
const connection = requireConnection(mailboxUid);
|
|
292
|
+
const { where, params } = buildSearchPredicates(parsed, mailboxUid);
|
|
293
|
+
const matchExpr = buildMatchExpression(parsed);
|
|
294
|
+
const hits: LocalSearchHit[] = [];
|
|
295
|
+
|
|
296
|
+
// A malformed MATCH string is a real, reachable case (FTS5's query syntax rejects some inputs
|
|
297
|
+
// `queryGrammar.ts` otherwise leaves untouched for the *server's* more lenient `websearch_to_tsquery`
|
|
298
|
+
// to handle) - fails soft to "this tier found nothing," matching how every other tier already
|
|
299
|
+
// degrades on its own per-candidate/per-provider failures, rather than breaking the whole search.
|
|
300
|
+
try {
|
|
301
|
+
// Fetches one extra row beyond `limit` so `hasMore` below can be determined without a second,
|
|
302
|
+
// separate COUNT(*) query - trimmed back off before returning.
|
|
303
|
+
const fetchLimit = limit + 1;
|
|
304
|
+
const sql = matchExpr
|
|
305
|
+
? `SELECT e.entity_uid, bm25(entities_fts, ${BM25_WEIGHTS_SQL}) AS rank,
|
|
306
|
+
snippet(entities_fts, 2, '', '', '…', 24) AS snip
|
|
307
|
+
FROM entities_fts f JOIN entities e ON e.rowid = f.rowid
|
|
308
|
+
WHERE entities_fts MATCH ? AND ${where}
|
|
309
|
+
ORDER BY rank LIMIT ? OFFSET ?`
|
|
310
|
+
: `SELECT e.entity_uid, 0 AS rank, NULL AS snip FROM entities e WHERE ${where}
|
|
311
|
+
ORDER BY e.date_for_sort DESC LIMIT ? OFFSET ?`;
|
|
312
|
+
const bindings = matchExpr ? [matchExpr, ...params, fetchLimit, offset] : [...params, fetchLimit, offset];
|
|
313
|
+
for await (const stmt of connection.sqlite3.statements(connection.db, sql)) {
|
|
314
|
+
connection.sqlite3.bind_collection(stmt, bindings);
|
|
315
|
+
while ((await connection.sqlite3.step(stmt)) === SQLite.SQLITE_ROW) {
|
|
316
|
+
hits.push({
|
|
317
|
+
entityUid: connection.sqlite3.column(stmt, 0) as string,
|
|
318
|
+
score: connection.sqlite3.column(stmt, 1) as number,
|
|
319
|
+
snippet: (connection.sqlite3.column(stmt, 2) as string | null) ?? undefined,
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
} catch {
|
|
324
|
+
return { hits: [], hasMore: false };
|
|
325
|
+
}
|
|
326
|
+
const hasMore = hits.length > limit;
|
|
327
|
+
if (hasMore) {
|
|
328
|
+
hits.length = limit;
|
|
329
|
+
}
|
|
330
|
+
return { hits, hasMore };
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
async function coverage(mailboxUid: string): Promise<Coverage> {
|
|
334
|
+
const connection = requireConnection(mailboxUid);
|
|
335
|
+
return {
|
|
336
|
+
indexedFrom: await oldestDateForSort(connection),
|
|
337
|
+
indexedCount: await entityCount(connection),
|
|
338
|
+
building: (await readMeta(connection, "building")) === "1",
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
async function setWindow({ mailboxUid, timeFloorMonths, byteBudgetBytes }: SetWindowParams): Promise<void> {
|
|
343
|
+
const connection = requireConnection(mailboxUid);
|
|
344
|
+
await writeMeta(connection, "time_floor_months", String(timeFloorMonths));
|
|
345
|
+
await writeMeta(connection, "byte_budget", String(byteBudgetBytes));
|
|
346
|
+
// A lowered budget must shrink the window immediately, not just gate future inserts (spec §11 "the
|
|
347
|
+
// client MUST reduce the window rather than fail writes when the budget is reached").
|
|
348
|
+
await applyEviction(connection);
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
async function setBuilding(mailboxUid: string, building: boolean): Promise<void> {
|
|
352
|
+
const connection = requireConnection(mailboxUid);
|
|
353
|
+
await writeMeta(connection, "building", building ? "1" : "0");
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/** Closes the connection (if open) and deletes the mailbox's entire OPFS directory - the spec's "MUST be
|
|
357
|
+
* destroyed on the same events that destroy private keys" (§11), and also the discard side of
|
|
358
|
+
* "discarded and rebuilt" on corruption/schema-version invalidation. Goes around SQLite/the VFS entirely
|
|
359
|
+
* for the deletion itself (there's no VFS-level "delete everything" primitive) - safe only because the
|
|
360
|
+
* connection is already closed at this point, so nothing else holds these files open. */
|
|
361
|
+
async function destroy(mailboxUid: string): Promise<void> {
|
|
362
|
+
await closeConnection(mailboxUid);
|
|
363
|
+
const root = await navigator.storage.getDirectory();
|
|
364
|
+
await root.removeEntry(poolNameFor(mailboxUid), { recursive: true }).catch(() => undefined);
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/**
|
|
368
|
+
* Proves the full encrypted round trip, not just a single write-then-read within one open connection
|
|
369
|
+
* (which could pass even if encryption/decryption were silently no-ops): writes a row, **closes the
|
|
370
|
+
* SQLite connection and drops it from `connections`**, then re-`init()`s the same mailbox from scratch -
|
|
371
|
+
* a real close/reopen through `EncryptingVFS`, `AccessHandlePoolVFS`, and OPFS, not merely reading back
|
|
372
|
+
* from an in-memory cache - and confirms the row (and a `bm25()`-ranked `MATCH` query against it) both
|
|
373
|
+
* still work after that reopen.
|
|
374
|
+
*/
|
|
375
|
+
async function selfTest(params: InitParams): Promise<{ matchedAfterReopen: string[] }> {
|
|
376
|
+
await init(params);
|
|
377
|
+
const before = requireConnection(params.mailboxUid);
|
|
378
|
+
await before.sqlite3.exec(before.db, "DELETE FROM entities");
|
|
379
|
+
for await (const stmt of before.sqlite3.statements(
|
|
380
|
+
before.db,
|
|
381
|
+
"INSERT INTO entities (entity_type, entity_uid, mailbox_uid, date_for_sort, subject, body) VALUES (?, ?, ?, ?, ?, ?)",
|
|
382
|
+
)) {
|
|
383
|
+
before.sqlite3.bind_collection(stmt, [
|
|
384
|
+
"message",
|
|
385
|
+
"spike-1",
|
|
386
|
+
params.mailboxUid,
|
|
387
|
+
new Date().toISOString(),
|
|
388
|
+
"Quarterly budget review",
|
|
389
|
+
"This round-trips through EncryptingVFS: written before a close, read back after a reopen.",
|
|
390
|
+
]);
|
|
391
|
+
await before.sqlite3.step(stmt);
|
|
392
|
+
}
|
|
393
|
+
await closeConnection(params.mailboxUid);
|
|
394
|
+
|
|
395
|
+
await init(params);
|
|
396
|
+
const after = requireConnection(params.mailboxUid);
|
|
397
|
+
const matchedAfterReopen: string[] = [];
|
|
398
|
+
for await (const stmt of after.sqlite3.statements(
|
|
399
|
+
after.db,
|
|
400
|
+
`SELECT e.entity_uid FROM entities_fts f JOIN entities e ON e.rowid = f.rowid
|
|
401
|
+
WHERE entities_fts MATCH 'budget' ORDER BY bm25(entities_fts, ${BM25_WEIGHTS_SQL})`,
|
|
402
|
+
)) {
|
|
403
|
+
while ((await after.sqlite3.step(stmt)) === SQLite.SQLITE_ROW) {
|
|
404
|
+
matchedAfterReopen.push(after.sqlite3.column(stmt, 0) as string);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
return { matchedAfterReopen };
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
self.addEventListener("message", (event: MessageEvent<LocalIndexRequest>) => {
|
|
411
|
+
const { id, method, params } = event.data;
|
|
412
|
+
void (async () => {
|
|
413
|
+
try {
|
|
414
|
+
let result: unknown;
|
|
415
|
+
switch (method) {
|
|
416
|
+
case "ping":
|
|
417
|
+
result = "pong";
|
|
418
|
+
break;
|
|
419
|
+
case "init":
|
|
420
|
+
await init(params as InitParams);
|
|
421
|
+
result = undefined;
|
|
422
|
+
break;
|
|
423
|
+
case "indexEntities":
|
|
424
|
+
await indexEntities(params as IndexEntitiesParams);
|
|
425
|
+
result = undefined;
|
|
426
|
+
break;
|
|
427
|
+
case "removeEntity":
|
|
428
|
+
await removeEntity(params as RemoveEntityParams);
|
|
429
|
+
result = undefined;
|
|
430
|
+
break;
|
|
431
|
+
case "search":
|
|
432
|
+
result = await search(params as SearchParams);
|
|
433
|
+
break;
|
|
434
|
+
case "coverage":
|
|
435
|
+
result = await coverage(params as string);
|
|
436
|
+
break;
|
|
437
|
+
case "setWindow":
|
|
438
|
+
await setWindow(params as SetWindowParams);
|
|
439
|
+
result = undefined;
|
|
440
|
+
break;
|
|
441
|
+
case "setBuilding": {
|
|
442
|
+
const { mailboxUid, building } = params as SetBuildingParams;
|
|
443
|
+
await setBuilding(mailboxUid, building);
|
|
444
|
+
result = undefined;
|
|
445
|
+
break;
|
|
446
|
+
}
|
|
447
|
+
case "destroy":
|
|
448
|
+
await destroy(params as string);
|
|
449
|
+
result = undefined;
|
|
450
|
+
break;
|
|
451
|
+
case "selfTest":
|
|
452
|
+
result = await selfTest(params as InitParams);
|
|
453
|
+
break;
|
|
454
|
+
default:
|
|
455
|
+
throw new Error(`Unknown localIndexWorker method: ${method satisfies never}`);
|
|
456
|
+
}
|
|
457
|
+
const response: LocalIndexResponse = { id, ok: true, result };
|
|
458
|
+
self.postMessage(response);
|
|
459
|
+
} catch (err) {
|
|
460
|
+
const response: LocalIndexResponse = { id, ok: false, error: err instanceof Error ? err.message : String(err) };
|
|
461
|
+
self.postMessage(response);
|
|
462
|
+
}
|
|
463
|
+
})();
|
|
464
|
+
});
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// Copyright (C) 2026 Jean-Philippe Steinmetz
|
|
3
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
5
|
+
/**
|
|
6
|
+
* Tier 2 (`specs/search.md` §6) - the local encrypted index's public entry point, mirroring
|
|
7
|
+
* `@rapidmx/react-shared`'s `searchTier3.ts#searchEncryptedCandidates()` in shape: given a parsed query
|
|
8
|
+
* and this session's unlocked keys, return already-scored, already-sourced results plus how much of the
|
|
9
|
+
* mailbox the local index actually covers right now.
|
|
10
|
+
*
|
|
11
|
+
* Returns `{ results: [], coverage: undefined }` - never throws, never rejects - both when `unlocked` is
|
|
12
|
+
* absent (nothing is indexed without a master key to derive the index key from, the same "silently
|
|
13
|
+
* contributes nothing" degradation `searchTier3.ts`'s own doc comment describes for its identical case)
|
|
14
|
+
* and when anything about the local index itself fails (Worker/WASM/OPFS unsupported or unavailable in
|
|
15
|
+
* this browser, a corrupted index mid-rebuild, a query timeout). This tier's own storage engine is
|
|
16
|
+
* strictly best-effort infrastructure Tier 1 doesn't depend on - a caller `Promise.all()`-ing this
|
|
17
|
+
* alongside Tier 1/Tier 3 (`apps/www/index.tsx`'s own search orchestration) must never see the *whole
|
|
18
|
+
* search* fail just because this one, most novel piece had a bad moment.
|
|
19
|
+
*/
|
|
20
|
+
import type { UnlockedKeys } from "@rapidmx/react-shared/crypto/keySession.js";
|
|
21
|
+
import type { ParsedSearchQuery } from "@rapidmx/react-shared/search/queryGrammar.js";
|
|
22
|
+
import type { SearchResult } from "@rapidmx/react-shared/search/searchApi.js";
|
|
23
|
+
import { deriveLocalIndexKey } from "./localIndexKey.js";
|
|
24
|
+
import { getLocalCoverage, initLocalIndex, searchLocal } from "./localIndexRpcClient.js";
|
|
25
|
+
import type { Coverage } from "./localIndexWorker.js";
|
|
26
|
+
|
|
27
|
+
export interface Tier2SearchOutcome {
|
|
28
|
+
results: SearchResult[];
|
|
29
|
+
/** `undefined` when `unlocked` was absent - there's no local index to report coverage for at all in
|
|
30
|
+
* that case, distinct from a real, empty (just-built) index. */
|
|
31
|
+
coverage?: Coverage;
|
|
32
|
+
/** `true` when at least one more match exists beyond `results` - §8's composite pagination cursor's
|
|
33
|
+
* own per-tier continuation signal. Always `false` alongside the `unlocked`-absent/error
|
|
34
|
+
* degradations below - there is nothing more to page through either way. */
|
|
35
|
+
hasMore: boolean;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Ensures this mailbox's local index connection is open before it's queried - idempotent
|
|
39
|
+
* (`localIndexWorker.ts`'s own `init()` no-ops for an already-open mailbox), so callers never need to
|
|
40
|
+
* coordinate with `LocalIndexLifecycle.tsx`'s own unlock-triggered `init()` call; whichever runs first
|
|
41
|
+
* wins, and the other is a cheap no-op. */
|
|
42
|
+
async function ensureInitialized(mailboxUid: string, unlocked: UnlockedKeys): Promise<void> {
|
|
43
|
+
const indexKey = await deriveLocalIndexKey(unlocked.masterKey, mailboxUid);
|
|
44
|
+
await initLocalIndex({ mailboxUid, indexKey });
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export async function searchLocalIndex(
|
|
48
|
+
mailboxUid: string,
|
|
49
|
+
parsed: ParsedSearchQuery,
|
|
50
|
+
unlocked: UnlockedKeys | undefined,
|
|
51
|
+
limit = 50,
|
|
52
|
+
offset = 0,
|
|
53
|
+
): Promise<Tier2SearchOutcome> {
|
|
54
|
+
if (!unlocked) {
|
|
55
|
+
return { results: [], hasMore: false };
|
|
56
|
+
}
|
|
57
|
+
try {
|
|
58
|
+
await ensureInitialized(mailboxUid, unlocked);
|
|
59
|
+
const [page, coverage] = await Promise.all([searchLocal(mailboxUid, parsed, limit, offset), getLocalCoverage(mailboxUid)]);
|
|
60
|
+
const results: SearchResult[] = page.hits.map((hit) => ({
|
|
61
|
+
entityType: "message",
|
|
62
|
+
entityUid: hit.entityUid,
|
|
63
|
+
// Negated: bm25()'s own convention is "more negative is a better match" (SQLite), the
|
|
64
|
+
// opposite of every other score this codebase merges (Postgres ts_rank, OpenSearch _score,
|
|
65
|
+
// and normalizeServerScores() itself all treat "higher is better"). Negating here, once,
|
|
66
|
+
// keeps that convention uniform for the caller (apps/www/index.tsx's merge layer) - it never
|
|
67
|
+
// needs to know this tier's underlying scoring function works backwards from the others.
|
|
68
|
+
score: -hit.score,
|
|
69
|
+
snippet: hit.snippet,
|
|
70
|
+
source: "local",
|
|
71
|
+
metadataOnly: false,
|
|
72
|
+
}));
|
|
73
|
+
return { results, coverage, hasMore: page.hasMore };
|
|
74
|
+
} catch {
|
|
75
|
+
// See this function's own doc comment - a broken local index degrades to "nothing to contribute
|
|
76
|
+
// this time," never a rejected search.
|
|
77
|
+
return { results: [], hasMore: false };
|
|
78
|
+
}
|
|
79
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
2
|
+
// Copyright (C) 2026 Jean-Philippe Steinmetz
|
|
3
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
///////////////////////////////////////////////////////////////////////////////
|
|
5
|
+
/**
|
|
6
|
+
* `@journeyapps/wa-sqlite` ships real `.d.ts` files for its top-level module and most `src/examples/*`
|
|
7
|
+
* VFS implementations, but not for `src/FacadeVFS.js` (the base class for hand-written VFS
|
|
8
|
+
* implementations) or `src/examples/AccessHandlePoolVFS.js` (the OPFS SAH Pool VFS this codebase's
|
|
9
|
+
* `EncryptingVFS` wraps) - both used directly by `localIndexVFS.ts`. Declared here, minimally, covering
|
|
10
|
+
* only the members this codebase actually calls.
|
|
11
|
+
*/
|
|
12
|
+
declare module "@journeyapps/wa-sqlite/src/FacadeVFS.js" {
|
|
13
|
+
import * as VFS from "@journeyapps/wa-sqlite/src/VFS.js";
|
|
14
|
+
|
|
15
|
+
/** Convenience base class for a JavaScript VFS - see the real source's own doc comment. Subclasses
|
|
16
|
+
* override the `jXxx` methods (JS-friendly wrappers) rather than the raw `xXxx` C-callback methods. */
|
|
17
|
+
export class FacadeVFS extends VFS.Base {
|
|
18
|
+
constructor(name: string, module: unknown);
|
|
19
|
+
hasAsyncMethod(methodName: string): boolean;
|
|
20
|
+
getFilename(pFile: number): string;
|
|
21
|
+
jOpen(filename: string | null, pFile: number, flags: number, pOutFlags: DataView): number | Promise<number>;
|
|
22
|
+
jDelete(filename: string, syncDir: number): number | Promise<number>;
|
|
23
|
+
jAccess(filename: string, flags: number, pResOut: DataView): number | Promise<number>;
|
|
24
|
+
jFullPathname(filename: string, zOut: Uint8Array): number | Promise<number>;
|
|
25
|
+
jClose(pFile: number): number | Promise<number>;
|
|
26
|
+
jRead(pFile: number, pData: Uint8Array, iOffset: number): number | Promise<number>;
|
|
27
|
+
jWrite(pFile: number, pData: Uint8Array, iOffset: number): number | Promise<number>;
|
|
28
|
+
jTruncate(pFile: number, size: number): number | Promise<number>;
|
|
29
|
+
jSync(pFile: number, flags: number): number | Promise<number>;
|
|
30
|
+
jFileSize(pFile: number, pSize64: DataView): number | Promise<number>;
|
|
31
|
+
jSectorSize(pFile: number): number;
|
|
32
|
+
jDeviceCharacteristics(pFile: number): number;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
declare module "@journeyapps/wa-sqlite/src/examples/AccessHandlePoolVFS.js" {
|
|
37
|
+
import { FacadeVFS } from "@journeyapps/wa-sqlite/src/FacadeVFS.js";
|
|
38
|
+
|
|
39
|
+
/** The OPFS "SAH Pool" VFS - synchronous, works with the plain (non-Asyncify) SQLite WASM build.
|
|
40
|
+
* `create()` is the intended entry point (constructs, then awaits internal readiness). */
|
|
41
|
+
export class AccessHandlePoolVFS extends FacadeVFS {
|
|
42
|
+
static create(name: string, module: unknown): Promise<AccessHandlePoolVFS>;
|
|
43
|
+
}
|
|
44
|
+
}
|