@rapidmx/web-client 0.35.0 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/apps/escrow/matters/[uid].tsx +4 -3
  2. package/apps/shared/calendar/sealedEvents.ts +80 -0
  3. package/apps/shared/components/admin/settings/EncryptionPolicyForm.tsx +61 -3
  4. package/apps/shared/components/contacts/contactCardData.ts +2 -1
  5. package/apps/shared/components/layout/KeyEnrollmentGate.tsx +23 -1
  6. package/apps/shared/components/layout/SharedMailboxKeyManager.tsx +91 -0
  7. package/apps/shared/components/layout/UnlockPromptProvider.tsx +34 -0
  8. package/apps/shared/components/mail/ConversationList.tsx +1 -1
  9. package/apps/shared/components/mail/MessageDetailPane.tsx +35 -13
  10. package/apps/shared/components/mail/compose/draftResume.ts +28 -3
  11. package/apps/shared/components/mail/compose/quotedBody.ts +4 -3
  12. package/apps/shared/components/mail/invite/inviteStore.ts +7 -1
  13. package/apps/shared/components/mail/layout/MailShell.tsx +2 -0
  14. package/apps/shared/components/mail/reading/useMessageActions.ts +4 -1
  15. package/apps/shared/components/settings/KeyExportSetting.tsx +170 -0
  16. package/apps/shared/components/settings/PolicyLockedNotice.tsx +23 -0
  17. package/apps/shared/components/settings/ZeroAccessSetting.tsx +117 -0
  18. package/apps/shared/mail/decryptedMessages.ts +11 -3
  19. package/apps/shared/mail/outbox/sendJob.ts +2 -2
  20. package/apps/shared/mail/reportNotices.ts +3 -1
  21. package/apps/shared/mail/zeroAccessSource.ts +27 -0
  22. package/apps/shared/search/LocalIndexLifecycle.tsx +31 -0
  23. package/apps/shared/search/localIndexBuilder.ts +27 -1
  24. package/apps/shared/search/localIndexRpcClient.ts +8 -4
  25. package/apps/shared/search/localIndexSchema.ts +2 -2
  26. package/apps/shared/search/localIndexTransport.ts +2 -1
  27. package/apps/shared/search/localIndexWorker.ts +6 -4
  28. package/apps/shared/search/zeroAccessIndexSync.ts +487 -0
  29. package/apps/www/calendar/index.tsx +176 -56
  30. package/apps/www/index.tsx +10 -3
  31. package/apps/www/settings/encryption/index.tsx +5 -0
  32. package/apps/www/settings/privacy/index.tsx +2 -1
  33. package/dist/apps/escrow/matters/[uid].js +2 -2
  34. package/dist/apps/shared/calendar/sealedEvents.d.ts +13 -0
  35. package/dist/apps/shared/calendar/sealedEvents.js +54 -0
  36. package/dist/apps/shared/components/admin/settings/EncryptionPolicyForm.js +18 -2
  37. package/dist/apps/shared/components/contacts/contactCardData.js +2 -1
  38. package/dist/apps/shared/components/layout/KeyEnrollmentGate.js +21 -2
  39. package/dist/apps/shared/components/layout/SharedMailboxKeyManager.d.ts +15 -0
  40. package/dist/apps/shared/components/layout/SharedMailboxKeyManager.js +72 -0
  41. package/dist/apps/shared/components/layout/UnlockPromptProvider.d.ts +3 -0
  42. package/dist/apps/shared/components/layout/UnlockPromptProvider.js +32 -0
  43. package/dist/apps/shared/components/mail/ConversationList.js +1 -1
  44. package/dist/apps/shared/components/mail/MessageDetailPane.js +34 -13
  45. package/dist/apps/shared/components/mail/compose/draftResume.d.ts +2 -0
  46. package/dist/apps/shared/components/mail/compose/draftResume.js +28 -3
  47. package/dist/apps/shared/components/mail/compose/quotedBody.d.ts +1 -1
  48. package/dist/apps/shared/components/mail/compose/quotedBody.js +4 -3
  49. package/dist/apps/shared/components/mail/invite/inviteStore.js +6 -1
  50. package/dist/apps/shared/components/mail/layout/MailShell.js +2 -1
  51. package/dist/apps/shared/components/mail/reading/useMessageActions.js +4 -1
  52. package/dist/apps/shared/components/settings/KeyExportSetting.d.ts +13 -0
  53. package/dist/apps/shared/components/settings/KeyExportSetting.js +88 -0
  54. package/dist/apps/shared/components/settings/PolicyLockedNotice.d.ts +11 -0
  55. package/dist/apps/shared/components/settings/PolicyLockedNotice.js +9 -0
  56. package/dist/apps/shared/components/settings/ZeroAccessSetting.d.ts +14 -0
  57. package/dist/apps/shared/components/settings/ZeroAccessSetting.js +71 -0
  58. package/dist/apps/shared/mail/decryptedMessages.d.ts +1 -1
  59. package/dist/apps/shared/mail/decryptedMessages.js +11 -3
  60. package/dist/apps/shared/mail/outbox/sendJob.js +2 -2
  61. package/dist/apps/shared/mail/reportNotices.js +3 -1
  62. package/dist/apps/shared/mail/zeroAccessSource.d.ts +8 -0
  63. package/dist/apps/shared/mail/zeroAccessSource.js +26 -0
  64. package/dist/apps/shared/search/LocalIndexLifecycle.d.ts +2 -0
  65. package/dist/apps/shared/search/LocalIndexLifecycle.js +28 -0
  66. package/dist/apps/shared/search/localIndexBuilder.d.ts +5 -0
  67. package/dist/apps/shared/search/localIndexBuilder.js +24 -2
  68. package/dist/apps/shared/search/localIndexRpcClient.d.ts +1 -1
  69. package/dist/apps/shared/search/localIndexRpcClient.js +6 -4
  70. package/dist/apps/shared/search/localIndexSchema.d.ts +1 -1
  71. package/dist/apps/shared/search/localIndexSchema.js +2 -2
  72. package/dist/apps/shared/search/localIndexTransport.d.ts +2 -1
  73. package/dist/apps/shared/search/localIndexWorker.d.ts +2 -0
  74. package/dist/apps/shared/search/localIndexWorker.js +4 -4
  75. package/dist/apps/shared/search/zeroAccessIndexSync.d.ts +71 -0
  76. package/dist/apps/shared/search/zeroAccessIndexSync.js +382 -0
  77. package/dist/apps/www/calendar/index.js +41 -15
  78. package/dist/apps/www/index.js +10 -3
  79. package/dist/apps/www/settings/encryption/index.js +3 -1
  80. package/dist/apps/www/settings/privacy/index.js +1 -1
  81. package/dist/lib/admin/escrowAccessRequestsApi.d.ts +6 -0
  82. package/dist/lib/calendar/calendarApi.d.ts +6 -0
  83. package/dist/lib/calendar/inviteApi.js +27 -5
  84. package/dist/lib/calendar/sealedInvites.d.ts +11 -0
  85. package/dist/lib/calendar/sealedInvites.js +28 -0
  86. package/dist/lib/crypto/keyExport.d.ts +54 -0
  87. package/dist/lib/crypto/keyExport.js +161 -0
  88. package/dist/lib/crypto/keySession.d.ts +45 -1
  89. package/dist/lib/crypto/keySession.js +93 -2
  90. package/dist/lib/crypto/keyvaultApi.d.ts +88 -3
  91. package/dist/lib/crypto/keyvaultApi.js +32 -0
  92. package/dist/lib/crypto/messageSecurity.d.ts +30 -1
  93. package/dist/lib/crypto/messageSecurity.js +80 -4
  94. package/dist/lib/crypto/mime.d.ts +5 -0
  95. package/dist/lib/crypto/mime.js +25 -0
  96. package/dist/lib/crypto/sharedMailboxKeys.d.ts +50 -0
  97. package/dist/lib/crypto/sharedMailboxKeys.js +264 -0
  98. package/dist/lib/crypto/zeroAccessKey.d.ts +32 -0
  99. package/dist/lib/crypto/zeroAccessKey.js +66 -0
  100. package/dist/lib/mail/conversationsApi.d.ts +6 -0
  101. package/dist/lib/mail/mailApi.d.ts +30 -2
  102. package/dist/lib/mail/mailApi.js +5 -1
  103. package/dist/lib/search/zeroAccessIndexApi.d.ts +81 -0
  104. package/dist/lib/search/zeroAccessIndexApi.js +35 -0
  105. package/lib/admin/escrowAccessRequestsApi.ts +6 -0
  106. package/lib/calendar/calendarApi.ts +3 -0
  107. package/lib/calendar/inviteApi.ts +28 -5
  108. package/lib/calendar/sealedInvites.ts +33 -0
  109. package/lib/crypto/keyExport.ts +229 -0
  110. package/lib/crypto/keySession.ts +117 -3
  111. package/lib/crypto/keyvaultApi.ts +125 -3
  112. package/lib/crypto/messageSecurity.ts +110 -3
  113. package/lib/crypto/mime.ts +26 -0
  114. package/lib/crypto/sharedMailboxKeys.ts +324 -0
  115. package/lib/crypto/zeroAccessKey.ts +97 -0
  116. package/lib/mail/conversationsApi.ts +3 -0
  117. package/lib/mail/mailApi.ts +33 -3
  118. package/lib/search/zeroAccessIndexApi.ts +119 -0
  119. package/package.json +3 -3
@@ -429,9 +429,9 @@ async function removeEntity(connection, entityUid) {
429
429
  }
430
430
  /** Re-points an indexed message at a new folder in place (archive, cancel-scheduled-send). Clears
431
431
  * `entity_version` so the next build pass re-validates the row against the server. */
432
- async function moveEntity(connection, entityUid, folderUid) {
433
- for await (const stmt of connection.sqlite3.statements(connection.db, "UPDATE entities SET folder_uid = ?, entity_version = NULL WHERE entity_uid = ?")) {
434
- connection.sqlite3.bind_collection(stmt, [folderUid, entityUid]);
432
+ async function moveEntity(connection, entityUid, folderUid, entityVersion) {
433
+ for await (const stmt of connection.sqlite3.statements(connection.db, "UPDATE entities SET folder_uid = ?, entity_version = ? WHERE entity_uid = ?")) {
434
+ connection.sqlite3.bind_collection(stmt, [folderUid, entityVersion ?? null, entityUid]);
435
435
  await connection.sqlite3.step(stmt);
436
436
  }
437
437
  }
@@ -636,7 +636,7 @@ export async function handleRequest(method, params) {
636
636
  }
637
637
  case "moveEntity": {
638
638
  const p = params;
639
- return runExclusive(p.mailboxUid, () => withConnection(p.mailboxUid, (c) => moveEntity(c, p.entityUid, p.folderUid)));
639
+ return runExclusive(p.mailboxUid, () => withConnection(p.mailboxUid, (c) => moveEntity(c, p.entityUid, p.folderUid, p.entityVersion)));
640
640
  }
641
641
  case "indexedVersions": {
642
642
  const p = params;
@@ -0,0 +1,71 @@
1
+ import type { UnlockedKeys } from "../../../lib/crypto/keySession.js";
2
+ import { type SegmentManifest, type ZeroAccessIndexDocument } from "../../../lib/search/zeroAccessIndexApi.js";
3
+ import { type ApiClient } from "../../../lib/util/api.js";
4
+ import type { LocalIndexEntity } from "./localIndexSchema.js";
5
+ /** The segment format this client writes and reads. */
6
+ export declare const SEGMENT_SCHEMA_VERSION = 1;
7
+ /** How far before its last pass the live tail asks again, so an entry committed late is never skipped. */
8
+ export declare const TAIL_OVERLAP_MS: number;
9
+ /** At most this many weeks are compacted per pass, so a large backlog is worked through over several passes. */
10
+ export declare const MAX_COMPACTIONS_PER_PASS = 4;
11
+ /** How many times `uploadClientDocuments()` re-reads a week's client segment after another device replaced it first. */
12
+ export declare const CLIENT_UPLOAD_ATTEMPTS = 3;
13
+ /** What a pass did, for the caller's logging and tests. */
14
+ export interface ZeroAccessSyncResult {
15
+ segmentsImported: number;
16
+ /** Client segments (end-to-end mail other devices indexed) imported. */
17
+ clientSegmentsImported: number;
18
+ entriesImported: number;
19
+ weeksCompacted: number;
20
+ /** Entries or segments that wouldn't decrypt with this device's keys (a key it doesn't have) - left as they are. */
21
+ unreadable: number;
22
+ }
23
+ /** Forgets what a pass remembered for `mailboxUid` (e.g. when its local index is destroyed). */
24
+ export declare function forgetZeroAccessSync(mailboxUid: string): void;
25
+ /** A document as a row of the local index. Folder and flags aren't in the document - the builder records the folder
26
+ * (`localIndexBuilder.ts`), and the upsert keeps what is already there. */
27
+ export declare function documentEntity(mailboxUid: string, doc: ZeroAccessIndexDocument): LocalIndexEntity;
28
+ /** One document of a client segment: a local index row of an end-to-end message, as the device that decrypted it built it. */
29
+ export interface ClientIndexDocument {
30
+ v: 1;
31
+ messageUid: string;
32
+ folderUid?: string;
33
+ /** ISO 8601. */
34
+ dateForSort: string;
35
+ participants: string;
36
+ flags: string;
37
+ hasAttachments: boolean;
38
+ subject?: string;
39
+ body?: string;
40
+ attachmentText?: string;
41
+ /** The message's version when it was decrypted (`localIndexBuilder.ts`'s `versionOf()`). */
42
+ entityVersion?: string;
43
+ }
44
+ /** A local index row as a client segment document. */
45
+ export declare function clientDocument(entity: LocalIndexEntity): ClientIndexDocument;
46
+ /** A client segment document as a row of the local index. */
47
+ export declare function clientDocumentEntity(mailboxUid: string, doc: ClientIndexDocument): LocalIndexEntity;
48
+ /** The ISO 8601 week (UTC) of `date`, as `YYYY-Www` - the week a message's segment is filed under (restapi's `isoWeekId()`). */
49
+ export declare function isoWeekId(date: Date): string;
50
+ /**
51
+ * Imports the client segments of `mailboxUid` (end-to-end mail indexed by another device - see this module's comment) not yet imported in
52
+ * their current generation, newest week first: how many. `manifest` saves reading it again; `generation` is the calling build pass's (see
53
+ * `GenerationParams`). Segments this device can't open are skipped.
54
+ */
55
+ export declare function importClientSegments(mailboxUid: string, unlocked: UnlockedKeys, client?: ApiClient, { manifest, generation }?: {
56
+ manifest?: SegmentManifest;
57
+ generation?: number;
58
+ }): Promise<number>;
59
+ /**
60
+ * Uploads end-to-end messages this device just decrypted and indexed (`entities`) to `mailboxUid`'s client segments, one per ISO week of
61
+ * `dateForSort`, each merged by message into the week's current segment and sealed to the active zero-access key: how many weeks were
62
+ * written. A week whose current segment this device can't read is left alone (merging would drop what is in it), as is one that already
63
+ * holds every document at the same version. Another device replacing a week first (`409`) is retried with its generation, up to
64
+ * `CLIENT_UPLOAD_ATTEMPTS` times. Does nothing without unlocked zero-access keys.
65
+ */
66
+ export declare function uploadClientDocuments(mailboxUid: string, unlocked: UnlockedKeys, entities: LocalIndexEntity[], client?: ApiClient): Promise<number>;
67
+ /**
68
+ * One sync pass for `mailboxUid` (see this module's comment). Does nothing without unlocked zero-access keys. Rejects when the manifest
69
+ * can't be read; any later failure ends the pass early with what it had done so far remembered.
70
+ */
71
+ export declare function syncZeroAccessIndex(mailboxUid: string, unlocked: UnlockedKeys, client?: ApiClient): Promise<ZeroAccessSyncResult>;
@@ -0,0 +1,382 @@
1
+ ///////////////////////////////////////////////////////////////////////////////
2
+ // Copyright (C) 2026 Jean-Philippe Steinmetz
3
+ // SPDX-License-Identifier: MPL-2.0
4
+ ///////////////////////////////////////////////////////////////////////////////
5
+ /**
6
+ * Keeps this device's local search index (Tier 2) in step with a zero-access mailbox's server index (`specs/search.md` §7-8): the
7
+ * server extracted every message's text at delivery and stores it only encrypted to the mailbox's zero-access key, as one sealed
8
+ * entry per message until a client compacts a closed week into a segment. A sync pass reads the manifest; imports each segment it
9
+ * hasn't imported in its current generation, newest weeks first (the worker's byte budget evicts the oldest); imports the pending entries
10
+ * written since its last pass (the live tail, asked from a little before then, skipping what it has); and compacts a few closed weeks
11
+ * that still have pending entries - it decrypts them and the week's current segment, merges them, seals the result to the mailbox's
12
+ * active zero-access key and uploads it as the next generation. A lost race (`409`) is left to the next pass.
13
+ *
14
+ * A segment (format version 1, `specs/search.md` §16) is gzipped JSON `{ v: 1, documents }` of the same documents the entries carry, so
15
+ * everything is imported through `indexLocalEntities()` - no new storage primitive, on web and on tauri alike.
16
+ *
17
+ * **Client segments (Source C).** End-to-end mail never reaches the server readable, so the server has nothing to index for it: the local
18
+ * index builder (`localIndexBuilder.ts`) decrypts it on one device and uploads what it built with `uploadClientDocuments()`, as `origin:
19
+ * "client"` segments sealed to the same zero-access key, merged into the week's current one. Every device imports them
20
+ * (`importClientSegments()`, run by the builder before it decrypts anything and by each sync pass), so end-to-end mail is decrypted for
21
+ * search once per mailbox rather than once per device - a row imported at the message's current version is not decrypted again.
22
+ *
23
+ * Nothing here ever sends the server anything it can read: entries are decrypted on this device, and a segment is sealed before upload.
24
+ */
25
+ import { fromBase64, toBase64 } from "../../../lib/crypto/encoding.js";
26
+ import { decryptEnvelopedDataWithKeys, encryptForRecipients } from "../../../lib/crypto/smime.js";
27
+ import { getSearchIndexManifest, getSearchIndexSegment, listSearchIndexEntries, putSearchIndexSegment, } from "../../../lib/search/zeroAccessIndexApi.js";
28
+ import { ApiRequestError } from "../../../lib/util/api.js";
29
+ import { getLocalCoverage, indexLocalEntities } from "./localIndexRpcClient.js";
30
+ /** The segment format this client writes and reads. */
31
+ export const SEGMENT_SCHEMA_VERSION = 1;
32
+ /** How far before its last pass the live tail asks again, so an entry committed late is never skipped. */
33
+ export const TAIL_OVERLAP_MS = 10 * 60 * 1000;
34
+ /** At most this many weeks are compacted per pass, so a large backlog is worked through over several passes. */
35
+ export const MAX_COMPACTIONS_PER_PASS = 4;
36
+ /** Entities handed to the worker per call. */
37
+ const IMPORT_BATCH = 200;
38
+ /** How many times `uploadClientDocuments()` re-reads a week's client segment after another device replaced it first. */
39
+ export const CLIENT_UPLOAD_ATTEMPTS = 3;
40
+ const stateKey = (mailboxUid) => `rapidmx:za-index-sync:${mailboxUid}`;
41
+ function loadState(mailboxUid) {
42
+ try {
43
+ const raw = localStorage.getItem(stateKey(mailboxUid));
44
+ const parsed = raw ? JSON.parse(raw) : undefined;
45
+ if (parsed && typeof parsed === "object" && typeof parsed.segments === "object") {
46
+ return parsed;
47
+ }
48
+ }
49
+ catch {
50
+ // Storage unavailable or corrupt: start over, which only costs a re-import.
51
+ }
52
+ return { segments: {} };
53
+ }
54
+ function saveState(mailboxUid, state) {
55
+ try {
56
+ localStorage.setItem(stateKey(mailboxUid), JSON.stringify(state));
57
+ }
58
+ catch {
59
+ // Not remembered: the next pass imports again, which is idempotent (entities are upserted).
60
+ }
61
+ }
62
+ /** Forgets what a pass remembered for `mailboxUid` (e.g. when its local index is destroyed). */
63
+ export function forgetZeroAccessSync(mailboxUid) {
64
+ try {
65
+ localStorage.removeItem(stateKey(mailboxUid));
66
+ }
67
+ catch {
68
+ // Nothing to forget.
69
+ }
70
+ }
71
+ async function transform(bytes, stream) {
72
+ const output = new Blob([bytes]).stream().pipeThrough(stream);
73
+ return new Uint8Array(await new Response(output).arrayBuffer());
74
+ }
75
+ const gunzip = (bytes) => transform(bytes, new DecompressionStream("gzip"));
76
+ const gzip = (bytes) => transform(bytes, new CompressionStream("gzip"));
77
+ /** Decrypts and gunzips `der` with `keys` and parses the JSON inside - `undefined` when no key opens it or it isn't JSON. */
78
+ async function openSealedJson(der, keys) {
79
+ try {
80
+ return JSON.parse(new TextDecoder().decode(await gunzip(await decryptEnvelopedDataWithKeys(der, keys))));
81
+ }
82
+ catch {
83
+ return undefined;
84
+ }
85
+ }
86
+ function isDocument(value) {
87
+ const doc = value;
88
+ return !!doc && doc.v === 1 && typeof doc.messageUid === "string" && typeof doc.subject === "string" && typeof doc.body === "string";
89
+ }
90
+ /** A document as a row of the local index. Folder and flags aren't in the document - the builder records the folder
91
+ * (`localIndexBuilder.ts`), and the upsert keeps what is already there. */
92
+ export function documentEntity(mailboxUid, doc) {
93
+ const participants = (doc.participants ?? []).join(" ");
94
+ const attachmentText = [...(doc.attachmentNames ?? []), ...(doc.attachmentText ?? [])].join("\n");
95
+ return {
96
+ entityType: "message",
97
+ entityUid: doc.messageUid,
98
+ mailboxUid,
99
+ dateForSort: new Date(Number.isFinite(doc.dateForSort) ? doc.dateForSort : 0).toISOString(),
100
+ participants,
101
+ flags: "",
102
+ hasAttachments: (doc.attachmentNames ?? []).length > 0,
103
+ subject: doc.subject,
104
+ body: doc.body,
105
+ attachmentText,
106
+ byteSize: doc.subject.length + doc.body.length + attachmentText.length + participants.length,
107
+ };
108
+ }
109
+ async function importDocuments(mailboxUid, docs) {
110
+ for (let i = 0; i < docs.length; i += IMPORT_BATCH) {
111
+ await indexLocalEntities(mailboxUid, docs.slice(i, i + IMPORT_BATCH).map((doc) => documentEntity(mailboxUid, doc)));
112
+ }
113
+ }
114
+ /** The documents of a segment generation that pass `accept`, or `undefined` when it won't open or is a format this client doesn't read. */
115
+ async function readSegmentAs(mailboxUid, segment, keys, accept, client) {
116
+ if (segment.schemaVersion !== SEGMENT_SCHEMA_VERSION) {
117
+ return undefined;
118
+ }
119
+ const bundle = (await openSealedJson(await getSearchIndexSegment(mailboxUid, segment, client), keys));
120
+ return bundle?.v === SEGMENT_SCHEMA_VERSION && Array.isArray(bundle.documents) ? bundle.documents.filter(accept) : undefined;
121
+ }
122
+ const readSegment = (mailboxUid, segment, keys, client) => readSegmentAs(mailboxUid, segment, keys, isDocument, client);
123
+ /** `documents` gzipped and sealed to `key` as a segment file, base64. */
124
+ async function sealSegment(documents, key) {
125
+ const file = await gzip(new TextEncoder().encode(JSON.stringify({ v: SEGMENT_SCHEMA_VERSION, documents })));
126
+ return toBase64(await encryptForRecipients(file, [key.certDer]));
127
+ }
128
+ function isClientDocument(value) {
129
+ const doc = value;
130
+ return !!doc && doc.v === 1 && typeof doc.messageUid === "string" && typeof doc.dateForSort === "string" && typeof doc.participants === "string";
131
+ }
132
+ /** A local index row as a client segment document. */
133
+ export function clientDocument(entity) {
134
+ return {
135
+ v: 1,
136
+ messageUid: entity.entityUid,
137
+ folderUid: entity.folderUid,
138
+ dateForSort: entity.dateForSort,
139
+ participants: entity.participants,
140
+ flags: entity.flags,
141
+ hasAttachments: entity.hasAttachments,
142
+ subject: entity.subject,
143
+ body: entity.body,
144
+ attachmentText: entity.attachmentText,
145
+ entityVersion: entity.entityVersion,
146
+ };
147
+ }
148
+ /** A client segment document as a row of the local index. */
149
+ export function clientDocumentEntity(mailboxUid, doc) {
150
+ const text = [doc.subject, doc.body, doc.attachmentText, doc.participants].reduce((total, value) => total + (value?.length ?? 0), 0);
151
+ return {
152
+ entityType: "message",
153
+ entityUid: doc.messageUid,
154
+ mailboxUid,
155
+ ...(doc.folderUid !== undefined ? { folderUid: doc.folderUid } : {}),
156
+ dateForSort: doc.dateForSort,
157
+ participants: doc.participants,
158
+ flags: typeof doc.flags === "string" ? doc.flags : "",
159
+ hasAttachments: doc.hasAttachments === true,
160
+ ...(doc.subject !== undefined ? { subject: doc.subject } : {}),
161
+ ...(doc.body !== undefined ? { body: doc.body } : {}),
162
+ ...(doc.attachmentText !== undefined ? { attachmentText: doc.attachmentText } : {}),
163
+ byteSize: text,
164
+ ...(doc.entityVersion !== undefined ? { entityVersion: doc.entityVersion } : {}),
165
+ };
166
+ }
167
+ /** The ISO 8601 week (UTC) of `date`, as `YYYY-Www` - the week a message's segment is filed under (restapi's `isoWeekId()`). */
168
+ export function isoWeekId(date) {
169
+ const day = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
170
+ // The Thursday of this ISO week decides its year (weeks start on Monday; getUTCDay() has Sunday as 0).
171
+ const weekday = day.getUTCDay() || 7;
172
+ day.setUTCDate(day.getUTCDate() + 4 - weekday);
173
+ const yearStart = new Date(Date.UTC(day.getUTCFullYear(), 0, 1));
174
+ const week = Math.ceil(((day.getTime() - yearStart.getTime()) / 86400000 + 1) / 7);
175
+ return `${day.getUTCFullYear()}-W${String(week).padStart(2, "0")}`;
176
+ }
177
+ /** The sync state, started over when the local index is empty (nothing remembered in it is there any more). */
178
+ async function currentState(mailboxUid) {
179
+ return (await getLocalCoverage(mailboxUid)).indexedCount === 0 ? { segments: {} } : loadState(mailboxUid);
180
+ }
181
+ /**
182
+ * Imports the client segments of `mailboxUid` (end-to-end mail indexed by another device - see this module's comment) not yet imported in
183
+ * their current generation, newest week first: how many. `manifest` saves reading it again; `generation` is the calling build pass's (see
184
+ * `GenerationParams`). Segments this device can't open are skipped.
185
+ */
186
+ export async function importClientSegments(mailboxUid, unlocked, client, { manifest, generation } = {}) {
187
+ const keys = unlocked.zeroAccessKeys;
188
+ if (!keys?.length) {
189
+ return 0;
190
+ }
191
+ const segments = (manifest ?? (await getSearchIndexManifest(mailboxUid, client))).segments.filter((segment) => segment.origin === "client");
192
+ const state = await currentState(mailboxUid);
193
+ let imported = 0;
194
+ for (const segment of segments.sort((a, b) => b.weekId.localeCompare(a.weekId))) {
195
+ const key = `${segment.weekId}:${segment.origin}`;
196
+ if (state.segments[key] === segment.etag) {
197
+ continue;
198
+ }
199
+ const docs = await readSegmentAs(mailboxUid, segment, keys, isClientDocument, client);
200
+ if (!docs) {
201
+ continue;
202
+ }
203
+ for (let i = 0; i < docs.length; i += IMPORT_BATCH) {
204
+ await indexLocalEntities(mailboxUid, docs.slice(i, i + IMPORT_BATCH).map((doc) => clientDocumentEntity(mailboxUid, doc)), generation);
205
+ }
206
+ state.segments[key] = segment.etag;
207
+ saveState(mailboxUid, state);
208
+ imported++;
209
+ }
210
+ return imported;
211
+ }
212
+ /**
213
+ * Uploads end-to-end messages this device just decrypted and indexed (`entities`) to `mailboxUid`'s client segments, one per ISO week of
214
+ * `dateForSort`, each merged by message into the week's current segment and sealed to the active zero-access key: how many weeks were
215
+ * written. A week whose current segment this device can't read is left alone (merging would drop what is in it), as is one that already
216
+ * holds every document at the same version. Another device replacing a week first (`409`) is retried with its generation, up to
217
+ * `CLIENT_UPLOAD_ATTEMPTS` times. Does nothing without unlocked zero-access keys.
218
+ */
219
+ export async function uploadClientDocuments(mailboxUid, unlocked, entities, client) {
220
+ const keys = unlocked.zeroAccessKeys;
221
+ if (!keys?.length || entities.length === 0) {
222
+ return 0;
223
+ }
224
+ const weeks = new Map();
225
+ for (const entity of entities) {
226
+ const weekId = isoWeekId(new Date(entity.dateForSort));
227
+ weeks.set(weekId, [...(weeks.get(weekId) ?? []), clientDocument(entity)]);
228
+ }
229
+ const state = loadState(mailboxUid);
230
+ let written = 0;
231
+ let manifest;
232
+ for (const [weekId, docs] of weeks) {
233
+ for (let attempt = 0; attempt < CLIENT_UPLOAD_ATTEMPTS; attempt++) {
234
+ manifest ?? (manifest = await getSearchIndexManifest(mailboxUid, client));
235
+ const current = manifest.segments.find((segment) => segment.origin === "client" && segment.weekId === weekId);
236
+ const previous = current ? await readSegmentAs(mailboxUid, current, keys, isClientDocument, client) : [];
237
+ if (!previous) {
238
+ break;
239
+ }
240
+ const merged = new Map(previous.map((doc) => [doc.messageUid, doc]));
241
+ if (docs.every((doc) => merged.has(doc.messageUid) && merged.get(doc.messageUid).entityVersion === doc.entityVersion)) {
242
+ break;
243
+ }
244
+ for (const doc of docs) {
245
+ merged.set(doc.messageUid, doc);
246
+ }
247
+ try {
248
+ const stored = await putSearchIndexSegment(mailboxUid, weekId, {
249
+ expectedGeneration: current?.generation ?? 0,
250
+ segment: await sealSegment([...merged.values()], keys[0]),
251
+ keyFingerprint: keys[0].fingerprint,
252
+ schemaVersion: SEGMENT_SCHEMA_VERSION,
253
+ origin: "client",
254
+ }, client);
255
+ // This device already holds every document of the new generation only if it held the previous one.
256
+ if (current && state.segments[`${weekId}:client`] === current.etag) {
257
+ state.segments[`${weekId}:client`] = stored.etag;
258
+ saveState(mailboxUid, state);
259
+ }
260
+ written++;
261
+ break;
262
+ }
263
+ catch (err) {
264
+ if (!(err instanceof ApiRequestError && err.status === 409)) {
265
+ throw err;
266
+ }
267
+ manifest = undefined;
268
+ }
269
+ }
270
+ }
271
+ return written;
272
+ }
273
+ /** Every pending entry of `week` (or, without one, written since `since`), paged through. */
274
+ async function listAllEntries(mailboxUid, filter, client) {
275
+ const entries = [];
276
+ let after;
277
+ do {
278
+ const page = await listSearchIndexEntries(mailboxUid, { ...filter, ...(after ? { after } : {}) }, client);
279
+ entries.push(...page.entries);
280
+ after = page.next;
281
+ } while (after);
282
+ return entries;
283
+ }
284
+ /** The documents of `entries` that open, with the uids of those entries; how many didn't. */
285
+ async function openEntries(entries, keys) {
286
+ const docs = [];
287
+ const uids = [];
288
+ let unreadable = 0;
289
+ for (const entry of entries) {
290
+ const doc = await openSealedJson(fromBase64(entry.sealed), keys);
291
+ if (isDocument(doc)) {
292
+ docs.push(doc);
293
+ uids.push(entry.uid);
294
+ }
295
+ else {
296
+ unreadable++;
297
+ }
298
+ }
299
+ return { docs, uids, unreadable };
300
+ }
301
+ /**
302
+ * One sync pass for `mailboxUid` (see this module's comment). Does nothing without unlocked zero-access keys. Rejects when the manifest
303
+ * can't be read; any later failure ends the pass early with what it had done so far remembered.
304
+ */
305
+ export async function syncZeroAccessIndex(mailboxUid, unlocked, client) {
306
+ const result = { segmentsImported: 0, clientSegmentsImported: 0, entriesImported: 0, weeksCompacted: 0, unreadable: 0 };
307
+ const keys = unlocked.zeroAccessKeys;
308
+ if (!keys?.length) {
309
+ return result;
310
+ }
311
+ const manifest = await getSearchIndexManifest(mailboxUid, client);
312
+ let state = await currentState(mailboxUid);
313
+ // Segments, newest week first.
314
+ const zeroAccessSegments = manifest.segments.filter((segment) => segment.origin === "zero-access");
315
+ for (const segment of [...zeroAccessSegments].sort((a, b) => b.weekId.localeCompare(a.weekId))) {
316
+ const key = `${segment.weekId}:${segment.origin}`;
317
+ if (state.segments[key] === segment.etag) {
318
+ continue;
319
+ }
320
+ const docs = await readSegment(mailboxUid, segment, keys, client);
321
+ if (!docs) {
322
+ result.unreadable++;
323
+ continue;
324
+ }
325
+ await importDocuments(mailboxUid, docs);
326
+ state.segments[key] = segment.etag;
327
+ result.segmentsImported++;
328
+ saveState(mailboxUid, state);
329
+ }
330
+ // End-to-end mail other devices indexed.
331
+ saveState(mailboxUid, state);
332
+ result.clientSegmentsImported = await importClientSegments(mailboxUid, unlocked, client, { manifest });
333
+ state = loadState(mailboxUid);
334
+ // The live tail.
335
+ const tailStart = new Date().toISOString();
336
+ const since = state.tailSince ? new Date(new Date(state.tailSince).getTime() - TAIL_OVERLAP_MS).toISOString() : undefined;
337
+ const tail = await openEntries(await listAllEntries(mailboxUid, since ? { since } : {}, client), keys);
338
+ await importDocuments(mailboxUid, tail.docs);
339
+ result.entriesImported += tail.docs.length;
340
+ result.unreadable += tail.unreadable;
341
+ state.tailSince = tailStart;
342
+ saveState(mailboxUid, state);
343
+ // Compaction of closed weeks.
344
+ const active = keys[0];
345
+ for (const week of manifest.weeks.filter((w) => w.compactable && w.pendingEntries > 0).slice(0, MAX_COMPACTIONS_PER_PASS)) {
346
+ const current = zeroAccessSegments.find((segment) => segment.weekId === week.weekId);
347
+ const previous = current ? await readSegment(mailboxUid, current, keys, client) : [];
348
+ if (!previous) {
349
+ // A generation this device can't read can't be folded into: leave the week to a device that can.
350
+ continue;
351
+ }
352
+ const pending = await openEntries(await listAllEntries(mailboxUid, { week: week.weekId }, client), keys);
353
+ if (pending.uids.length === 0) {
354
+ continue;
355
+ }
356
+ const merged = new Map(previous.map((doc) => [doc.messageUid, doc]));
357
+ for (const doc of pending.docs) {
358
+ merged.set(doc.messageUid, doc);
359
+ }
360
+ try {
361
+ const stored = await putSearchIndexSegment(mailboxUid, week.weekId, {
362
+ expectedGeneration: current?.generation ?? 0,
363
+ segment: await sealSegment([...merged.values()], active),
364
+ keyFingerprint: active.fingerprint,
365
+ schemaVersion: SEGMENT_SCHEMA_VERSION,
366
+ foldedEntryUids: pending.uids,
367
+ }, client);
368
+ // This device already holds every document of the new generation.
369
+ await importDocuments(mailboxUid, pending.docs);
370
+ state.segments[`${stored.weekId}:${stored.origin}`] = stored.etag;
371
+ saveState(mailboxUid, state);
372
+ result.weeksCompacted++;
373
+ }
374
+ catch (err) {
375
+ // Another device compacted the week first: the next pass imports its generation.
376
+ if (!(err instanceof ApiRequestError && err.status === 409)) {
377
+ throw err;
378
+ }
379
+ }
380
+ }
381
+ return result;
382
+ }