mindoodb 0.0.86 → 0.0.88

package/dist/core/BaseMindooDB.js

@@ -482,6 +482,16 @@ class BaseMindooDB {
         this.syncStoreChangesInFlight = null;
         /** Shared follow-up run for callers that arrive mid-sync. */
         this.syncStoreChangesQueued = null;
+        /**
+         * > 0 while this database hands a sealed document's wrap list to the tenant
+         * ({@link ingestSealed}). Unwrapping may resolve the own User-Key, which
+         * reads `directory` (grants, username hashes) and so calls
+         * {@link syncStoreChanges} on it. When this database *is* `directory` (a
+         * sealed admin document there, e.g. a mail gateway configuration sealed to
+         * the gateway), that call would wait for the very run that is waiting for
+         * it. Such nested calls return at once and read the index as it is.
+         */
+        this.sealedIngestDepth = 0;
         this.changeListeners = new Set();
         this.pendingChangeNotifications = new Map();
         this.changeNotifyTimer = null;
@@ -784,16 +794,97 @@ class BaseMindooDB {
      * — a peer sync — and therefore have to apply the admin/owner rules
      * themselves.
      *
-     * Deliberately the same check the load path uses, so an entry admitted here
-     * is one that will also materialize. The alternative, accepting on ingest
-     * and dropping on load, leaves the entry sitting in the store and pushes it
-     * on to the server at the next sync, where it is refused and fails the run.
+     * This is the server's write rule ({@link evaluateBuiltinWrite}), not the
+     * lenient load-path filter. The load path may fail open because whatever it
+     * reads already passed a server; an entry arriving from a peer has passed
+     * nothing, so an owner hash that cannot be resolved is a refusal here, as it
+     * is on the server. Everything this admits also passes the load path, so an
+     * admitted entry still materializes.
+     *
+     * `batch` is the rest of the delivery: a new document arrives as its
+     * `doc_create` plus later changes in one batch, and the create is what names
+     * the owner. Pass entries with their payload (`encryptedData`) where you have
+     * it; a `doc_create` of a userkey document is decrypted to read
+     * `username_hash`.
      *
      * Returns false for databases without a builtin invariant, so callers can
      * ask unconditionally.
      */
-    async violatesBuiltinWriteInvariant(entry) {
-        return this.shouldSkipLoadedEntryForBuiltin(entry);
+    async violatesBuiltinWriteInvariant(entry, batch = []) {
+        return (await this.builtinIngestDenial(entry, batch)) !== null;
+    }
+    /** Reason the builtin invariant refuses `entry` from a peer, or null. */
+    async builtinIngestDenial(entry, batch) {
+        const dbId = this.effectiveBuiltinDbId();
+        if (!(0, builtinDbInvariants_1.hasBuiltinWriteInvariant)(dbId)) {
+            return null;
+        }
+        const entryType = entry.entryType ?? "doc_change";
+        const op = (0, builtinDbInvariants_1.entryTypeToBuiltinOp)(entryType);
+        if (!op) {
+            // Not a document lifecycle entry (attachment chunks and the like). The
+            // admin-only rule still holds for `directory`; `userdirectory` has no
+            // rule for them, exactly as on the server.
+            if (dbId === types_1.DIRECTORY_DB_ID && !this.isAdminKey(entry.createdByPublicKey)) {
+                return "Admin-only database: only the admin key can modify data";
+            }
+            return null;
+        }
+        let documentUsernameHash = null;
+        let signerUsernameHash = null;
+        let creatorUsernameHash = null;
+        if (dbId === types_1.USER_DIRECTORY_DB_ID) {
+            // Fresh, not the load-path cache: that one remembers a miss for a key
+            // that has since been granted.
+            signerUsernameHash = await this.resolveSignerUsernameHash(entry.createdByPublicKey);
+            const createInBatch = entryType === "doc_create"
+                ? entry
+                : batch.find((candidate) => candidate.entryType === "doc_create" && candidate.docId === entry.docId);
+            if ((0, builtinDbInvariants_1.isPersonalUserdirectoryDocId)(entry.docId)) {
+                if (op === "doc_create") {
+                    creatorUsernameHash = signerUsernameHash;
+                }
+                else if (createInBatch) {
+                    creatorUsernameHash = await this.resolveSignerUsernameHash(createInBatch.createdByPublicKey);
+                }
+                else if (entry.docId) {
+                    creatorUsernameHash = await this.ensurePersonalDocCreatorHash(entry.docId);
+                }
+            }
+            else if (createInBatch?.encryptedData) {
+                documentUsernameHash = await this.usernameHashFromCreateEntry(createInBatch);
+            }
+            else if (entry.docId && op !== "doc_create") {
+                documentUsernameHash = await this.ensureUserdirectoryDocHash(entry.docId);
+            }
+        }
+        const decision = (0, builtinDbInvariants_1.evaluateBuiltinWrite)({
+            dbId,
+            op,
+            signerKey: entry.createdByPublicKey,
+            adminPublicKey: this.getAdminPublicKey(),
+            documentUsernameHash,
+            signerUsernameHash,
+            docId: entry.docId,
+            creatorUsernameHash,
+        });
+        return decision.allowed ? null : decision.reason;
+    }
+    isAdminKey(publicKey) {
+        return (0, DirectoryStateNode_1.signingKeysEqual)(publicKey, this.getAdminPublicKey());
+    }
+    /** `username_hash` out of a userkey document's `doc_create` payload. */
+    async usernameHashFromCreateEntry(entry) {
+        if (!entry.encryptedData || !entry.decryptionKeyId) {
+            return null;
+        }
+        try {
+            const decrypted = await this.tenant.decryptPayload(entry.encryptedData, entry.decryptionKeyId);
+            return (0, builtinDbInvariants_1.usernameHashFromCreateChangeBytes)(decrypted);
+        }
+        catch {
+            return null;
+        }
     }
     async ensureSignerUsernameHash(signerKey) {
         if (this.signerUsernameHashByKey.has(signerKey)) {
@@ -2828,6 +2919,93 @@ class BaseMindooDB {
         }
         return outcome;
     }
+    /**
+     * Request the hard delete of deleted documents in a personal database
+     * (`mmail_…`, `mmailout_…`; docs/accesscontrol.md §6.11, "Purge"). Writes a
+     * purge request document (`purgereq_…`, `$publicinfos`) that names the
+     * documents; push it like any change. The server accepts it only from the
+     * owner's devices, a service with a matching `accessPrefixes` grant or the
+     * tenant admin, and only for documents that are deleted; it then removes
+     * their history and refuses it from then on. Every replica (also this one)
+     * removes the documents when it next pulls the request
+     * ({@link applyPurgeRequests}).
+     *
+     * @returns The id of the purge request document.
+     */
+    async requestPurgeOfDeletedDocuments(docIds) {
+        this.assertWritable("requestPurgeOfDeletedDocuments");
+        const dbId = this.store.getId();
+        if (!(0, builtinDbInvariants_1.personalDatabaseOwner)(dbId)) {
+            throw new Error(`Purge requests are only for personal databases, not ${dbId}`);
+        }
+        const ids = Array.from(new Set(docIds.filter((id) => typeof id === "string" && id.length > 0)));
+        if (ids.length === 0)
+            throw new Error("No document ids to purge");
+        const deleted = new Set(await this.getDeletedDocumentIds());
+        for (const id of ids) {
+            if ((0, builtinDbInvariants_1.isPurgeRequestDocId)(id))
+                throw new Error(`A purge request cannot be purged: ${id}`);
+            if (!deleted.has(id))
+                throw new Error(`Document ${id} is not deleted; delete it before purging`);
+        }
+        const request = await this.createDocument({
+            idPrefix: builtinDbInvariants_1.PURGE_REQUEST_ID_PREFIX.slice(0, -1),
+            decryptionKeyId: types_1.PUBLIC_INFOS_KEY_ID,
+            initialValues: { form: builtinDbInvariants_1.PURGE_REQUEST_FORM, docIds: ids, requestedAt: Date.now() },
+        });
+        return request.getId();
+    }
+    /**
+     * The document ids the purge requests of this personal database name
+     * (§6.11). Empty for any other database. With `onlyAccepted`, only requests
+     * the server has accepted (their `doc_create` carries a witness receipt):
+     * a request that is still local, or that the server refused, purges
+     * nothing, so a refused request never loses data that stays on the server.
+     */
+    async getPurgeRequestedDocIds(options) {
+        const result = new Set();
+        if (!(0, builtinDbInvariants_1.personalDatabaseOwner)(this.store.getId()))
+            return result;
+        for (const requestId of await this.getAllDocumentIds({ idPrefix: builtinDbInvariants_1.PURGE_REQUEST_ID_PREFIX.slice(0, -1) })) {
+            if (options?.onlyAccepted) {
+                const metas = await this.store.findNewEntriesForDoc([], requestId);
+                const create = metas.find((m) => m.entryType === "doc_create");
+                if (!create?.receivedAt)
+                    continue;
+            }
+            try {
+                const data = (await this.getDocument(requestId)).getData();
+                if (data.form !== builtinDbInvariants_1.PURGE_REQUEST_FORM || !Array.isArray(data.docIds))
+                    continue;
+                for (const id of data.docIds) {
+                    const docId = String(id);
+                    if (docId && !(0, builtinDbInvariants_1.isPurgeRequestDocId)(docId))
+                        result.add(docId);
+                }
+            }
+            catch {
+                // Not readable here (no $publicinfos yet): picked up on a later run.
+            }
+        }
+        return result;
+    }
+    /**
+     * Execute the purge requests of this personal database locally (§6.11):
+     * remove every named document that still has entries here. Idempotent; runs
+     * after every docs pull of a personal database.
+     *
+     * @returns How many documents were removed in this run.
+     */
+    async applyPurgeRequests() {
+        this.assertWritable("applyPurgeRequests");
+        let removed = 0;
+        for (const docId of await this.getPurgeRequestedDocIds({ onlyAccepted: true })) {
+            const outcome = await this.purgeDocumentHistory(docId);
+            if (outcome.docEntriesPurged > 0 || outcome.attachmentChunksPurged > 0)
+                removed++;
+        }
+        return removed;
+    }
     /**
      * Drop everything this instance derived from a document whose store entries
      * were purged: the L1/L2 caches, summary values and full-text tokens (via
@@ -2907,7 +3085,7 @@ class BaseMindooDB {
             }
             const existingIndex = this.getDocIndexPosition(docId);
             const existing = existingIndex === undefined ? undefined : this.index[existingIndex];
-            await this.tenant.ingestSealedFromEntries?.(visibility.decryptionKeyId, metadataByDoc.get(docId));
+            await this.ingestSealed(visibility.decryptionKeyId, metadataByDoc.get(docId));
             const canRead = await this.tenant.hasDecryptionKey(visibility.decryptionKeyId);
             if (canRead) {
                 // This scan is the authority on visibility, so it also heals a tally
@@ -3187,6 +3365,10 @@ class BaseMindooDB {
      * {@link syncStoreChangesInFlight}).
      */
     async syncStoreChanges() {
+        if (this.sealedIngestDepth > 0) {
+            // Called back from our own sealed ingest (see sealedIngestDepth).
+            return;
+        }
         if (this.syncStoreChangesInFlight) {
             if (!this.syncStoreChangesQueued) {
                 // The running scan may have started before the entries this caller
@@ -3328,7 +3510,7 @@ class BaseMindooDB {
                             });
                             sealedIngestEntries = fullDocMetadata;
                         }
-                        await this.tenant.ingestSealedFromEntries?.(representativeEntry.decryptionKeyId, sealedIngestEntries);
+                        await this.ingestSealed(representativeEntry.decryptionKeyId, sealedIngestEntries);
                         const keyAvailable = await this.tenant.hasDecryptionKey(representativeEntry.decryptionKeyId);
                         if (!keyAvailable) {
                             this.logger.debug(`Skipping metadata-first index for doc ${docId} — decryption key "${representativeEntry.decryptionKeyId}" not available`);
@@ -4077,6 +4259,9 @@ class BaseMindooDB {
             }
             this.validateIdPrefixOption("createDocuments", options);
             this.validateAssumeUniqueIdOption("createDocuments", options);
+            if ((options.attachments?.length ?? 0) > 0) {
+                throw new Error("createDocuments: attachments are only supported by createDocument()");
+            }
             (0, changeAttribution_1.encodeChangeAttribution)(options.attribution);
         }
         const results = new Array(inputs.length).fill(null);
@@ -4255,6 +4440,10 @@ class BaseMindooDB {
                 throw new Error("createDocument: recipients is not supported together with a convergent custom id; " +
                     "pass assumeUniqueId for random ids.");
             }
+            if (useDeterministicSeed && (options.attachments?.length ?? 0) > 0) {
+                throw new Error("createDocument: attachments are not supported together with a convergent custom id; " +
+                    "pass assumeUniqueId, or add them with changeDoc().");
+            }
         }
         const signerPublicKey = await this.resolveSignerPublicKey(useCustomSigningKey ? signingKeyPair : undefined);
         const createUsernameHash = this.usernameHashFromRecord(Object.fromEntries(initialValueEntries));
@@ -4313,11 +4502,17 @@ class BaseMindooDB {
         else {
             this.logger.debug(`Creating initial Automerge change for document ${docId}`);
             const initialDoc = automerge_adapter_1.Automerge.init();
+            // Attachments of the first entry: chunks are written now, under the
+            // document's key, and referenced from the doc_create change below.
+            const createAttachments = [];
+            for (const attachment of options.attachments ?? []) {
+                createAttachments.push(await this.addAttachmentInternal(docId, attachment.data, attachment.fileName, attachment.mimeType, keyId, now));
+            }
             try {
                 newDoc = automerge_adapter_1.Automerge.change(initialDoc, changeMessage ? { message: changeMessage } : {}, (doc) => {
                     // Store metadata in the document payload
                     // We need to modify the document to ensure a change is created
-                    doc._attachments = [];
+                    doc._attachments = createAttachments.map((ref) => ({ ...ref }));
                     if (sealedCreate) {
                         (0, recipients_1.applyEncryptForCreate)(doc, sealedCreate.encryptFor);
                     }
@@ -4824,6 +5019,22 @@ class BaseMindooDB {
         });
         return this.wrapDocument(internalDoc);
     }
+    /**
+     * Hands a sealed document's newest wrap list to the tenant, with nested
+     * {@link syncStoreChanges} calls on this database short-circuited meanwhile
+     * (see {@link sealedIngestDepth}).
+     */
+    async ingestSealed(keyId, entries) {
+        if (!this.tenant.ingestSealedFromEntries)
+            return;
+        this.sealedIngestDepth++;
+        try {
+            await this.tenant.ingestSealedFromEntries(keyId, entries);
+        }
+        finally {
+            this.sealedIngestDepth--;
+        }
+    }
     /**
      * Unwrap a `$sealed:` DEK from entry recipient wraps into the tenant session
      * cache. `loadDocumentInternal` already does this; history, timestamp, and
@@ -4836,7 +5047,7 @@ class BaseMindooDB {
             return;
         const keyId = entries.find((entry) => entry.entryType === "doc_create")
             ?.decryptionKeyId ?? entries[0].decryptionKeyId;
-        await this.tenant.ingestSealedFromEntries?.(keyId, entries);
+        await this.ingestSealed(keyId, entries);
     }
     /**
      * Re-ingest the newest sealed wrap list. When this replica can no longer
@@ -5221,6 +5432,23 @@ class BaseMindooDB {
         // of fold order (under the trusted-time rule un-witnessed entries can sort
         // before the create, so we must not rely on seeing the create first).
         const createMeta = replayMeta.find((m) => m.entryType === "doc_create");
+        // A document this replica holds no key for (e.g. an admin document in
+        // `directory` encrypted with `default` or sealed to someone else, read by a
+        // `$publicinfos`-only member or the sync server) is not part of its feed,
+        // just as it is not part of its index; decrypting it would throw and stop
+        // the whole feed (and with it the directory state and access checks).
+        const feedKeyId = createMeta?.decryptionKeyId ?? replayMeta[0]?.decryptionKeyId;
+        if (feedKeyId && !(await this.tenant.hasDecryptionKey(feedKeyId))) {
+            // Sealed: the wrap list may not be ingested yet. Not for admin-only
+            // databases: ingesting there can read `directory`, whose state this
+            // feed may be building (the directory sync already ingested them).
+            if ((0, sealedTypes_1.isSealedKeyId)(feedKeyId) && !this._isAdminOnlyDb) {
+                await this.ingestSealed(feedKeyId, replayMeta);
+            }
+            if (!(await this.tenant.hasDecryptionKey(feedKeyId))) {
+                return;
+            }
+        }
         // Seed the running doc to the merge of the unchanged prefix.
         const prefix = ttSorted.slice(0, firstChangedIdx).map((r) => r.meta);
         const seed = await this.seedRevisionFold(docId, prefix, docAllMeta);
@@ -11968,7 +12196,7 @@ class BaseMindooDB {
             .map((em) => `${em.entryType}@${em.createdAt}`)
             .join(", ");
         this.logger.debug(`Entry types for ${docId}: ${entryTypes}`);
-        await this.tenant.ingestSealedFromEntries?.(allEntryMetadata.find((em) => em.entryType === "doc_create")
+        await this.ingestSealed(allEntryMetadata.find((em) => em.entryType === "doc_create")
             ?.decryptionKeyId ?? allEntryMetadata[0].decryptionKeyId, allEntryMetadata);
         const loadKeyId = allEntryMetadata.find((em) => em.entryType === "doc_create")
             ?.decryptionKeyId ?? allEntryMetadata[0].decryptionKeyId;
@@ -13275,9 +13503,19 @@ class BaseMindooDB {
         // erased history back (docs/accesscontrol.md §13). Never applied to the
         // directory, which carries the purge requests themselves.
         const dbId = this.store.getId();
+        // Personal databases carry their own purge requests (§6.11) on top.
+        const isPersonal = (0, builtinDbInvariants_1.personalDatabaseOwner)(dbId) !== null;
         const localStore = dbId === "directory"
             ? this.getStoreForKind(storeKind)
-            : (0, purgedDocumentGuard_1.guardStoreAgainstPurgedDocuments)(this.getStoreForKind(storeKind), () => this.tenant.getPurgedDocumentIds(dbId), this.logger);
+            : (0, purgedDocumentGuard_1.guardStoreAgainstPurgedDocuments)(this.getStoreForKind(storeKind), async () => {
+                const fromDirectory = await this.tenant.getPurgedDocumentIds(dbId);
+                if (!isPersonal)
+                    return fromDirectory;
+                const own = await this.getPurgeRequestedDocIds();
+                for (const id of fromDirectory)
+                    own.add(id);
+                return own;
+            }, this.logger);
         if (localStore.getId() !== remoteStore.getId() ||
             localStore.getStoreKind() !== remoteStore.getStoreKind()) {
             throw new Error(`[BaseMindooDB] Cannot pull entries from the incompatible store ${localStore.getId()}/${localStore.getStoreKind()}`);
@@ -13385,6 +13623,17 @@ class BaseMindooDB {
             await this.tenant.reconcileUserKeysSafe?.();
             await this.tenant.reconcileKeyDistributionsForCurrentUserSafe();
         }
+        // Personal databases: execute the purge requests the server accepted (§6.11).
+        if ((0, builtinDbInvariants_1.personalDatabaseOwner)(this.store.getId())) {
+            try {
+                const removed = await this.applyPurgeRequests();
+                if (removed > 0)
+                    this.logger.info(`Purged ${removed} document(s) by purge request`);
+            }
+            catch (error) {
+                this.logger.warn(`Applying purge requests failed: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
     }
     /**
      * Push changes to a remote content-addressed store or another MindooDB instance.