@rapidmx/web-client 0.35.0 → 0.37.0

package/apps/shared/search/localIndexRpcClient.ts

@@ -26,6 +26,11 @@
  * replacement. `searchTier2.ts` additionally races its own calls against a timeout, since a Worker that's
  * merely stuck (not crashed) posts no `error` event at all.
  *
+ * **A lock destroys the index**: `initLocalIndex()` subscribes once to the key session (`subscribeKeySession()`); when the keys of
+ * a mailbox whose index it opened are destroyed (an idle lock, a manual lock, a sign-out), that mailbox's build is cancelled and its
+ * index destroyed (spec section 11), whether or not a mail view (`LocalIndexLifecycle`) is still mounted to notice - after leaving Mail
+ * for Calendar, or in a host whose account switch unmounted the shell that opened it (`tauri-client`).
+ *
  * **Swappable transport**: everything above is specific to this one implementation - the Worker/OPFS-backed
  * default. Every function this module exports below (`initLocalIndex`, `searchLocal`, `destroyLocalIndex`, ...)
  * is a thin delegator to whichever `LocalIndexTransport` (`localIndexTransport.ts`) is "current," which defaults
@@ -38,6 +43,7 @@
  * plain exports, not part of the swappable interface - they name this one transport's own cross-tab/localStorage
  * mechanics, which a different transport has no reason to share.
  */
+import { subscribeKeySession } from "../../../lib/crypto/keySession.js";
 import type { ParsedSearchQuery } from "../../../lib/search/queryGrammar.js";
 import type {
     Coverage,
@@ -48,6 +54,7 @@ import type {
     IndexedVersionsParams,
     InitParams,
     LocalIndexRequest,
+    LocalIndexResetNotice,
     LocalIndexResponse,
     LocalSearchPage,
     MoveEntityParams,
@@ -60,6 +67,7 @@ import type {
 } from "./localIndexWorker.js";
 import type { LocalIndexEntity } from "./localIndexSchema.js";
 import { removeLocalIndexDirectories, removeLocalIndexDirectory } from "./localIndexStorage.js";
+import { forgetAllZeroAccessSync, forgetZeroAccessSync, forgetZeroAccessSyncExcept } from "./zeroAccessSyncState.js";
 import type { LocalIndexTransport, PruneLocalEntitiesOptions, SetLocalIndexBuildingCompletion } from "./localIndexTransport.js";
 
 let worker: Worker | undefined;
@@ -94,7 +102,12 @@ function getWorker(): Worker {
         // listener's own comment for why that guard matters.
         const created = new Worker(new URL("./localIndexWorker.js", import.meta.url), { type: "module" });
         worker = created;
-        created.addEventListener("message", (event: MessageEvent<LocalIndexResponse>) => {
+        created.addEventListener("message", (event: MessageEvent<LocalIndexResponse | LocalIndexResetNotice>) => {
+            if ("type" in event.data) {
+                // The Worker discarded a corrupted index: what sync passes remembered of it goes too, or a rebuilt one would skip it.
+                forgetZeroAccessSync(event.data.mailboxUid);
+                return;
+            }
             const entry = pending.get(event.data.id);
             if (!entry) {
                 return;
@@ -251,11 +264,19 @@ async function removeLocalEntityImpl(mailboxUid: string, entityUid: string): Pro
 
 /** Re-points one indexed message at its new folder (archive, cancel-scheduled-send) so `folder:`-scoped
  * local searches stay correct. Same no-op/never-rejects rules as `removeLocalEntityImpl()`. */
-async function moveLocalEntityImpl(mailboxUid: string, entityUid: string, folderUid: string): Promise<void> {
+async function moveLocalEntityImpl(mailboxUid: string, entityUid: string, folderUid: string, entityVersion?: string, flags?: string): Promise<void> {
     if (!initializedMailboxes.has(mailboxUid)) {
         return;
     }
-    await call("moveEntity", { mailboxUid, entityUid, folderUid } satisfies MoveEntityParams).catch(() => undefined);
+    await call("moveEntity", {
+        mailboxUid,
+        entityUid,
+        folderUid,
+        ...(entityVersion !== undefined ? { entityVersion } : {}),
+        ...(flags !== undefined ? { flags } : {}),
+    } satisfies MoveEntityParams).catch(
+        () => undefined,
+    );
 }
 
 function getIndexedVersionsImpl(mailboxUid: string, entityUids: string[]): Promise<Record<string, string>> {
@@ -288,6 +309,9 @@ function setLocalIndexBuildingImpl(mailboxUid: string, building: boolean, comple
  * when the index could not be removed, e.g. because another tab still has it open; the deletion is then
  * retried on the next page load. */
 async function destroyLocalIndexImpl(mailboxUid: string): Promise<boolean> {
+    // What zero-access sync passes remembered of it goes with it, here too, so a host wrapper destroying mailboxes one by one through this
+    // transport (`tauri-client`'s scoped sign-out) forgets theirs.
+    forgetZeroAccessSync(mailboxUid);
     updatePendingDeletions((current) => current.add(mailboxUid));
     try {
         await call("destroy", { mailboxUid, generation: nextLocalIndexGenerationImpl() } satisfies DestroyParams);
@@ -315,6 +339,9 @@ export const DESTROY_ALL_TIMEOUT_MS = 3_000;
  */
 async function destroyAllLocalIndexesImpl(timeoutMs: number): Promise<boolean> {
     signedOut = true;
+    // Here, in the transport, as for prune: a host wrapper that destroys only some accounts' indexes (`tauri-client`, several accounts in
+    // one origin) keeps the other accounts' sync state too.
+    forgetAllZeroAccessSync();
     writePendingDeletions([ALL_INDEXES]);
     try {
         getChannel()?.postMessage({ type: "sign-out" });
@@ -347,9 +374,13 @@ async function destroyAllLocalIndexesImpl(timeoutMs: number): Promise<boolean> {
 }
 
 /** Removes local indexes for mailboxes outside `accessibleMailboxUids` - e.g. left behind by a different
- * user who signed in on this device without signing out. Never rejects. */
+ * user who signed in on this device without signing out - and what zero-access sync passes remembered of them, as on a destroy. Both
+ * here, in the transport, so a host's wrapper that keeps other indexes (`tauri-client`, several accounts in one origin) keeps their sync
+ * state too. Never rejects. */
 async function pruneInaccessibleLocalIndexesImpl(accessibleMailboxUids: Iterable<string>): Promise<void> {
-    await removeLocalIndexDirectories(new Set(accessibleMailboxUids)).catch(() => undefined);
+    const accessible = new Set(accessibleMailboxUids);
+    forgetZeroAccessSyncExcept(accessible);
+    await removeLocalIndexDirectories(accessible).catch(() => undefined);
 }
 
 /**
@@ -394,8 +425,49 @@ export function nextLocalIndexGeneration(): number {
     return currentTransport.nextGeneration();
 }
 
-export function initLocalIndex(params: InitParams): Promise<void> {
-    return currentTransport.init(params);
+/** Every mailbox whose index `initLocalIndex()` opened, through whichever transport, and that wasn't destroyed since. */
+const openedMailboxes = new Set<string>();
+/** How many times each mailbox's keys were destroyed this page load - an `initLocalIndex()` that a lock overtook destroys what it opened. */
+const lockCounts = new Map<string, number>();
+let watchingLocks = false;
+
+/** Cancels `mailboxUid`'s build and destroys its index, as `LocalIndexLifecycle` does on a lock. Never rejects. */
+function destroyOnLock(mailboxUid: string): void {
+    // Stop a build pass still running with the destroyed keys; the destroy also makes the Worker reject anything that pass still sends.
+    // Loaded lazily (it imports this module, and carries the message security code).
+    import("./localIndexBuilder.js")
+        .then(({ cancelLocalIndexBuild }) => cancelLocalIndexBuild(mailboxUid))
+        .catch(() => undefined);
+    void destroyLocalIndex(mailboxUid);
+}
+
+/** Subscribes, once, to key destruction: see this module's doc comment ("A lock destroys the index"). */
+function watchKeyLocks(): void {
+    if (watchingLocks) {
+        return;
+    }
+    watchingLocks = true;
+    subscribeKeySession(({ mailboxUid, state }) => {
+        if (state !== "locked") {
+            return;
+        }
+        lockCounts.set(mailboxUid, (lockCounts.get(mailboxUid) ?? 0) + 1);
+        if (openedMailboxes.has(mailboxUid)) {
+            destroyOnLock(mailboxUid);
+        }
+    });
+}
+
+export async function initLocalIndex(params: InitParams): Promise<void> {
+    watchKeyLocks();
+    const locks = lockCounts.get(params.mailboxUid) ?? 0;
+    await currentTransport.init(params);
+    if ((lockCounts.get(params.mailboxUid) ?? 0) !== locks) {
+        // Its keys were destroyed while it opened: it never counted as open, so the lock's own destroy didn't reach it.
+        destroyOnLock(params.mailboxUid);
+        return;
+    }
+    openedMailboxes.add(params.mailboxUid);
 }
 
 export function indexLocalEntities(mailboxUid: string, entities: LocalIndexEntity[], generation?: number): Promise<IndexEntitiesResult> {
@@ -409,9 +481,15 @@ export function removeLocalEntity(mailboxUid: string, entityUid: string): Promis
 }
 
 /** Re-points one indexed message at its new folder (archive, cancel-scheduled-send) so `folder:`-scoped
- * local searches stay correct. Same no-op/never-rejects rules as `removeLocalEntity()`. */
-export function moveLocalEntity(mailboxUid: string, entityUid: string, folderUid: string): Promise<void> {
-    return currentTransport.moveEntity(mailboxUid, entityUid, folderUid);
+ * local searches stay correct. Same no-op/never-rejects rules as `removeLocalEntity()`. `flags` (`,flag,flag,`) sets the row's flags
+ * too - a zero-access row's, which nothing else records. */
+export function moveLocalEntity(mailboxUid: string, entityUid: string, folderUid: string, entityVersion?: string, flags?: string): Promise<void> {
+    if (flags !== undefined) {
+        return currentTransport.moveEntity(mailboxUid, entityUid, folderUid, entityVersion, flags);
+    }
+    return entityVersion !== undefined
+        ? currentTransport.moveEntity(mailboxUid, entityUid, folderUid, entityVersion)
+        : currentTransport.moveEntity(mailboxUid, entityUid, folderUid);
 }
 
 export function getIndexedVersions(mailboxUid: string, entityUids: string[]): Promise<Record<string, string>> {
@@ -448,19 +526,25 @@ export function setLocalIndexBuilding(mailboxUid: string, building: boolean, com
  * resolves `false` when the index could not be removed - see the current transport's own doc comment for what
  * that means for it (the Worker transport also logs why, and retries on the next page load). */
 export function destroyLocalIndex(mailboxUid: string): Promise<boolean> {
+    openedMailboxes.delete(mailboxUid);
+    // What zero-access sync passes remembered importing into it goes with it, whatever the transport.
+    forgetZeroAccessSync(mailboxUid);
     return currentTransport.destroy(mailboxUid);
 }
 
 /** Destroys **every** local index this transport knows of - not just the mailboxes this session opened - e.g.
  * on sign-out. Never rejects. Resolves `true` only if everything was removed within `timeoutMs`. */
 export function destroyAllLocalIndexes(timeoutMs = DESTROY_ALL_TIMEOUT_MS): Promise<boolean> {
+    // `openedMailboxes` is left as it is: a host transport may keep other accounts' indexes open (`tauri-client`), and a lock of one
+    // already destroyed finds nothing to close.
     return currentTransport.destroyAll(timeoutMs);
 }
 
 /** Removes local indexes for mailboxes outside `accessibleMailboxUids` - e.g. left behind by a different
- * user who signed in on this device without signing out. Never rejects. */
+ * user who signed in on this device without signing out. The transport decides what goes, zero-access sync state included (the
+ * Worker transport forgets it with the index; a host wrapper that skips the prune keeps both). Never rejects. */
 export function pruneInaccessibleLocalIndexes(accessibleMailboxUids: Iterable<string>): Promise<void> {
-    return currentTransport.pruneInaccessible(accessibleMailboxUids);
+    return currentTransport.pruneInaccessible(new Set(accessibleMailboxUids));
 }
 
 /** Retries every deletion an earlier session couldn't finish. Never rejects. */

package/apps/shared/search/localIndexSchema.ts

@@ -114,17 +114,33 @@ export interface LocalIndexEntity {
     entityVersion?: string;
 }
 
+/** The change marker a message's row stores (`LocalIndexEntity.entityVersion`): a new server `version` (flags, content) or a folder move
+ * both invalidate the indexed copy. */
+export function messageRowVersion(message: { version: number; folderUid: string }): string {
+    return `${message.version}:${message.folderUid}`;
+}
+
+/** A message's set flags as a row stores them. Leading/trailing comma so `flags LIKE '%,x,%'` (`buildSearchPredicates()`) matches
+ * correctly even for the first/last flag in the list. */
+export function messageRowFlags(message: { flags: object }): string {
+    const set = Object.entries(message.flags as Record<string, unknown>)
+        .filter(([, value]) => value)
+        .map(([key]) => key);
+    return `,${set.join(",")},`;
+}
+
 /** `entity_uid` upsert - `ON CONFLICT` (SQLite's UPSERT syntax) rather than a separate delete-then-insert,
  * so re-indexing an already-present message (a flag changed, a folder move) updates it in place and the
- * `entities_au` trigger keeps `entities_fts` in sync automatically. */
+ * `entities_au` trigger keeps `entities_fts` in sync automatically. A row written without a folder, flags or version (a zero-access
+ * document, which carries none) keeps the ones recorded: re-importing it must not make it look unrecorded again. */
 export const UPSERT_ENTITY_SQL = `
 INSERT INTO entities (entity_type, entity_uid, mailbox_uid, folder_uid, date_for_sort, participants, flags, has_attachments, subject, body, attachment_text, byte_size, entity_version)
 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 ON CONFLICT(entity_uid) DO UPDATE SET
-    folder_uid = excluded.folder_uid, date_for_sort = excluded.date_for_sort, participants = excluded.participants,
-    flags = excluded.flags, has_attachments = excluded.has_attachments, subject = excluded.subject,
+    folder_uid = COALESCE(excluded.folder_uid, entities.folder_uid), date_for_sort = excluded.date_for_sort, participants = excluded.participants,
+    flags = CASE WHEN excluded.flags = '' THEN entities.flags ELSE excluded.flags END, has_attachments = excluded.has_attachments, subject = excluded.subject,
     body = excluded.body, attachment_text = excluded.attachment_text, byte_size = excluded.byte_size,
-    entity_version = excluded.entity_version
+    entity_version = COALESCE(excluded.entity_version, entities.entity_version)
 `;
 
 /** Bind values for `UPSERT_ENTITY_SQL`, in column order - kept alongside it so the two can never drift

package/apps/shared/search/localIndexTransport.ts

@@ -59,7 +59,9 @@ export interface LocalIndexTransport {
     /** Drops one indexed entity. */
     removeEntity(mailboxUid: string, entityUid: string): Promise<void>;
     /** Re-points one indexed entity at its new folder. */
-    moveEntity(mailboxUid: string, entityUid: string, folderUid: string): Promise<void>;
+    /** `entityVersion`, when given, is recorded with the folder (a transport that can't record it leaves the row's version cleared), and
+     * `flags` replaces the row's flags (a transport that can't record them leaves them as they are). */
+    moveEntity(mailboxUid: string, entityUid: string, folderUid: string, entityVersion?: string, flags?: string): Promise<void>;
     /** The indexed version marker (`Message.version` + folder) currently stored for each of the given entity uids. */
     indexedVersions(mailboxUid: string, entityUids: string[]): Promise<Record<string, string>>;
     /** Removes indexed rows outside `keepEntityUids` (and `since`/`options.folderUids`'s own scope). Returns how many rows were pruned. */
@@ -77,7 +79,9 @@ export interface LocalIndexTransport {
     /** Destroys every index this transport knows of, e.g. on sign-out. Never rejects; resolves `true` only if
      * everything was removed within `timeoutMs`. */
     destroyAll(timeoutMs: number): Promise<boolean>;
-    /** Removes indexes for mailboxes outside `accessibleMailboxUids`. Never rejects. */
+    /** Removes indexes for mailboxes outside `accessibleMailboxUids`, and what zero-access sync remembered of each one it removes
+     * (`forgetZeroAccessSyncExcept()`; `pruneInaccessibleLocalIndexes()` leaves that to the transport, so a host can keep both). Never
+     * rejects. */
     pruneInaccessible(accessibleMailboxUids: Iterable<string>): Promise<void>;
     /** Retries any deletion an earlier session couldn't finish. Never rejects. */
     retryPendingDeletions(): Promise<void>;

package/apps/shared/search/localIndexWorker.ts

@@ -72,6 +72,12 @@ export type LocalIndexResponse =
     | { id: number; ok: true; result: unknown }
     | { id: number; ok: false; error: string };
 
+/** Posted unprompted when a corrupted index was discarded (`resetCorruptedConnection()`), so the main thread forgets what it remembered of it. */
+export interface LocalIndexResetNotice {
+    type: "reset";
+    mailboxUid: string;
+}
+
 /**
  * Carried by every call a build pass makes (`init`, `setWindow`, `setBuilding`, `indexEntities`,
  * `pruneEntities`). Generations come from one monotonic counter on the main thread
@@ -124,6 +130,10 @@ export interface MoveEntityParams {
     mailboxUid: string;
     entityUid: string;
     folderUid: string;
+    /** Recorded as the row's `entity_version`; cleared (the row then counts as changed) when absent. */
+    entityVersion?: string;
+    /** The row's flags (`,flag,flag,`), set too when given - a zero-access row, whose imported document carries none. */
+    flags?: string;
 }
 
 export interface IndexedVersionsParams {
@@ -519,6 +529,7 @@ async function withCurrentConnection<T>(params: GenerationParams & { mailboxUid:
 async function resetCorruptedConnection(mailboxUid: string, connection: OpenConnection): Promise<void> {
     console.warn(`localIndexWorker: local search index for mailbox ${mailboxUid} is corrupted; discarding and rebuilding.`);
     connections.delete(mailboxUid);
+    self.postMessage({ type: "reset", mailboxUid } satisfies LocalIndexResetNotice);
     await closeRawConnection(connection).catch(() => undefined);
     try {
         await removeLocalIndexDirectory(mailboxUid);
@@ -635,31 +646,39 @@ async function removeEntity(connection: OpenConnection, entityUid: string): Prom
 }
 
 /** Re-points an indexed message at a new folder in place (archive, cancel-scheduled-send). Clears
- * `entity_version` so the next build pass re-validates the row against the server. */
-async function moveEntity(connection: OpenConnection, entityUid: string, folderUid: string): Promise<void> {
-    for await (const stmt of connection.sqlite3.statements(connection.db, "UPDATE entities SET folder_uid = ?, entity_version = NULL WHERE entity_uid = ?")) {
-        connection.sqlite3.bind_collection(stmt, [folderUid, entityUid]);
+ * `entity_version` so the next build pass re-validates the row against the server. Sets its flags too when `flags` is given. */
+async function moveEntity(connection: OpenConnection, entityUid: string, folderUid: string, entityVersion?: string, flags?: string): Promise<void> {
+    const [sql, values] =
+        flags === undefined
+            ? ["UPDATE entities SET folder_uid = ?, entity_version = ? WHERE entity_uid = ?", [folderUid, entityVersion ?? null, entityUid]]
+            : ["UPDATE entities SET folder_uid = ?, entity_version = ?, flags = ? WHERE entity_uid = ?", [folderUid, entityVersion ?? null, flags, entityUid]];
+    for await (const stmt of connection.sqlite3.statements(connection.db, sql)) {
+        connection.sqlite3.bind_collection(stmt, values);
         await connection.sqlite3.step(stmt);
     }
 }
 
+/** How many uids one `indexedVersions()` statement binds. */
+export const INDEXED_VERSIONS_BATCH = 500;
+
 /** `entity_version` for whichever of `entityUids` are already indexed - one batched query per builder page,
  * so a rebuild can skip re-fetching/decrypting unchanged messages. */
 async function indexedVersions(connection: OpenConnection, entityUids: string[]): Promise<Record<string, string>> {
     const versions: Record<string, string> = {};
-    if (entityUids.length === 0) {
-        return versions;
-    }
-    const placeholders = entityUids.map(() => "?").join(", ");
-    for await (const stmt of connection.sqlite3.statements(
-        connection.db,
-        `SELECT entity_uid, entity_version FROM entities WHERE entity_uid IN (${placeholders})`,
-    )) {
-        connection.sqlite3.bind_collection(stmt, entityUids);
-        while ((await connection.sqlite3.step(stmt)) === SQLite.SQLITE_ROW) {
-            const version = connection.sqlite3.column(stmt, 1) as string | null;
-            if (version !== null) {
-                versions[connection.sqlite3.column(stmt, 0) as string] = version;
+    // In statements of at most `INDEXED_VERSIONS_BATCH` uids: SQLite refuses one binding more than its variable limit (32766 here).
+    for (let i = 0; i < entityUids.length; i += INDEXED_VERSIONS_BATCH) {
+        const batch: string[] = entityUids.slice(i, i + INDEXED_VERSIONS_BATCH);
+        const placeholders = batch.map(() => "?").join(", ");
+        for await (const stmt of connection.sqlite3.statements(
+            connection.db,
+            `SELECT entity_uid, entity_version FROM entities WHERE entity_uid IN (${placeholders})`,
+        )) {
+            connection.sqlite3.bind_collection(stmt, batch);
+            while ((await connection.sqlite3.step(stmt)) === SQLite.SQLITE_ROW) {
+                const version = connection.sqlite3.column(stmt, 1) as string | null;
+                if (version !== null) {
+                    versions[connection.sqlite3.column(stmt, 0) as string] = version;
+                }
             }
         }
     }
@@ -871,7 +890,7 @@ export async function handleRequest(method: LocalIndexRequest["method"], params:
         }
         case "moveEntity": {
             const p = params as MoveEntityParams;
-            return runExclusive(p.mailboxUid, () => withConnection(p.mailboxUid, (c) => moveEntity(c, p.entityUid, p.folderUid)));
+            return runExclusive(p.mailboxUid, () => withConnection(p.mailboxUid, (c) => moveEntity(c, p.entityUid, p.folderUid, p.entityVersion, p.flags)));
         }
         case "indexedVersions": {
             const p = params as IndexedVersionsParams;

package/apps/shared/search/searchTier2.ts

@@ -18,6 +18,7 @@
  * search* fail just because this one, most novel piece had a bad moment.
  */
 import type { UnlockedKeys } from "../../../lib/crypto/keySession.js";
+import { KeysLockedError } from "../../../lib/crypto/masterKey.js";
 import type { ParsedSearchQuery } from "../../../lib/search/queryGrammar.js";
 import type { SearchResult } from "../../../lib/search/searchApi.js";
 import { deriveLocalIndexKey } from "./localIndexKey.js";
@@ -41,6 +42,10 @@ export interface Tier2SearchOutcome {
  * wins, and the other is a cheap no-op. */
 async function ensureInitialized(mailboxUid: string, unlocked: UnlockedKeys): Promise<void> {
     const indexKey = await deriveLocalIndexKey(unlocked.masterKey, mailboxUid);
+    // Locked during the derive, which still succeeds: open nothing with the destroyed keys (the lock's own destroy found nothing open).
+    if (unlocked.destroyed) {
+        throw new KeysLockedError();
+    }
     await initLocalIndex({ mailboxUid, indexKey });
 }