@peerbit/document 13.0.43 → 13.1.0

package/dist/src/program.js

@@ -33,17 +33,20 @@ var __runInitializers = (this && this.__runInitializers) || function (thisArg, i
     return useValue ? value : void 0;
 };
 import { BorshError, field, serialize, variant, } from "@dao-xyz/borsh";
-import { AccessError, SignatureWithKey } from "@peerbit/crypto";
-import { NotFoundError, } from "@peerbit/document-interface";
+import { AccessError, SignatureWithKey, } from "@peerbit/crypto";
+import { Context, NotFoundError, } from "@peerbit/document-interface";
+import { extractDocumentFieldSimple, initializeDocumentRust, planDocumentContext, planDocumentContextBatch, tryPlanDocumentContext, tryPlanDocumentContextBatch, } from "./native-rust.js";
 import * as indexerTypes from "@peerbit/indexer-interface";
-import { Entry, EntryType, } from "@peerbit/log";
+import { Entry, EntryType, LamportClock, ShallowEntry, ShallowMeta, Timestamp, entryV0PlainPayloadDataFromStorage, } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
 import { Program } from "@peerbit/program";
 import { SharedLog, } from "@peerbit/shared-log";
 import { MAX_BATCH_SIZE } from "./constants.js";
 import { BORSH_ENCODING_OPERATION, DeleteOperation, PutOperation, PutWithKeyOperation, coerceDeleteOperation, isDeleteOperation, isPutOperation, } from "./operation.js";
+import { createCanPerformPolicyEvaluator, createCanPerformDeletePolicyEvaluator, getCanPerformPolicyDescriptor, canPerformPolicyDeleteFieldPaths, canPerformPolicyNeedsDeleteValue, canPerformPolicyNeedsPreviousEntries, canPerformPolicyPutNeedsEntryPublicKeys, canPerformPolicySignedByFieldPaths, } from "./policy.js";
 import { isResultIndexedValue } from "./result-shape.js";
-import { DocumentIndex, INDEX_CONTEXT_SHAPE, coerceWithContext, coerceWithIndexed, } from "./search.js";
+import { DocumentIndex, INDEX_CONTEXT_SHAPE, coerceWithContext, coerceWithIndexed, coerceWithLazyIndexed, encodeContextSuffix as encodeDocumentContextSuffix, } from "./search.js";
+import { getDocumentTransformDescriptor, } from "./transform.js";
 const logger = loggerFn("peerbit:program:document");
 const warn = logger.newScope("warn");
 export class OperationError extends Error {
@@ -51,6 +54,271 @@ export class OperationError extends Error {
         super(message);
     }
 }
+export class NativeDocumentModeError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "NativeDocumentModeError";
+    }
+}
+const isPromiseLike = (value) => !!value && typeof value.then === "function";
+const mapMaybePromise = (value, fn) => (isPromiseLike(value) ? value.then(fn) : fn(value));
+const bytesEqual = (left, right) => {
+    if (left.byteLength !== right.byteLength) {
+        return false;
+    }
+    for (let i = 0; i < left.byteLength; i++) {
+        if (left[i] !== right[i]) {
+            return false;
+        }
+    }
+    return true;
+};
+const PUT_OPERATION_PREFIX_LENGTH = 6;
+const encodePutOperationPayload = (data) => {
+    const encoded = new Uint8Array(PUT_OPERATION_PREFIX_LENGTH + data.byteLength);
+    encoded[0] = 0;
+    encoded[1] = 3;
+    const view = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength);
+    view.setUint32(2, data.byteLength, true);
+    encoded.set(data, PUT_OPERATION_PREFIX_LENGTH);
+    return encoded;
+};
+const toContextBigInt = (value) => typeof value === "bigint" ? value : BigInt(value);
+const NATIVE_LOCAL_PUT_OPTIONS = Object.freeze({
+    replicate: false,
+    target: "none",
+});
+const NATIVE_LOCAL_UNIQUE_PUT_OPTIONS = Object.freeze({
+    unique: true,
+    replicate: false,
+    target: "none",
+});
+const cachedNativeLocalPutOptions = (options) => {
+    if (!options) {
+        return NATIVE_LOCAL_PUT_OPTIONS;
+    }
+    let empty = true;
+    for (const key in options) {
+        empty = false;
+        if (key !== "unique") {
+            return;
+        }
+    }
+    if (empty) {
+        return NATIVE_LOCAL_PUT_OPTIONS;
+    }
+    return options.unique === true ? NATIVE_LOCAL_UNIQUE_PUT_OPTIONS : undefined;
+};
+class CompatDocumentBackend {
+    putImpl;
+    putManyImpl;
+    deleteImpl;
+    constructor(putImpl, putManyImpl, deleteImpl) {
+        this.putImpl = putImpl;
+        this.putManyImpl = putManyImpl;
+        this.deleteImpl = deleteImpl;
+    }
+    put(doc, options) {
+        return this.putImpl(doc, options);
+    }
+    putMany(docs, options) {
+        return this.putManyImpl(docs, options);
+    }
+    del(id, options) {
+        return this.deleteImpl(id, options);
+    }
+}
+class NativeDocumentBackend {
+    context;
+    constructor(context) {
+        this.context = context;
+    }
+    put(doc, options) {
+        this.context.assertPlainPutSupported(doc, options);
+        const putOptions = this.context.normalizePutOptions(options);
+        const prepared = this.context.preparePlainPut(doc);
+        const commit = (existing, useNativeExistingDocumentContext = !putOptions?.unique &&
+            existing === undefined, requiredPreviousSignerPublicKey) => {
+            const nextEntry = existing
+                ? this.context.getNextFromIndexedContext(existing)
+                : undefined;
+            if (existing && !nextEntry) {
+                throw this.context.nativeModeError("requires indexed document context for native put");
+            }
+            const next = nextEntry ? [nextEntry] : [];
+            return mapMaybePromise(this.context.commitNativeDocumentAppend({
+                document: prepared.document,
+                key: prepared.key,
+                documentBytes: prepared.encodedDocument,
+                operationPayloadBytes: prepared.operationPayloadBytes,
+                next: next,
+                skipMissingNextJoin: true,
+                resolveTrimmedEntries: this.context.shouldResolveTrimmedEntries(),
+                options: putOptions,
+                unique: putOptions?.unique,
+                useNativeExistingDocumentContext,
+                requiredPreviousSignerPublicKey,
+                existing,
+            }), (documentAppendCommit) => mapMaybePromise(this.context.handlePreparedPlainPutCommit(documentAppendCommit), () => {
+                this.context.keepEntry(documentAppendCommit.append.hash);
+                return {
+                    get entry() {
+                        return documentAppendCommit.entry;
+                    },
+                    removed: documentAppendCommit.removed,
+                };
+            }));
+        };
+        const assertPolicyAndCommit = (existingContext, previousSignerPublicKey) => {
+            const existingHead = this.context.getIndexedContextHead(existingContext);
+            const nativePreviousSignerPublicKey = previousSignerPublicKey ??
+                (existingHead
+                    ? this.context.getNativeEntrySignerPublicKeys([existingHead])?.[0]
+                    : undefined);
+            return mapMaybePromise(this.context.assertPlainPutPolicySupported(prepared.document, existingContext, nativePreviousSignerPublicKey), () => commit(existingContext));
+        };
+        if (!putOptions?.unique &&
+            this.context.plainPutPolicyNeedsExistingContext()) {
+            const requiredPreviousSignerPublicKey = this.context.getNativeAppendRequiredPreviousSignerPublicKey();
+            if (requiredPreviousSignerPublicKey) {
+                return commit(undefined, true, requiredPreviousSignerPublicKey);
+            }
+            const nativeContexts = this.context.getNativeIndexedContextsAndPreviousSignerPublicKeys([
+                prepared.key,
+            ]);
+            if (nativeContexts) {
+                return assertPolicyAndCommit(nativeContexts.contexts[0] ?? null, nativeContexts.publicKeys[0]);
+            }
+            const nativePreviousSigner = this.context.getNativePreviousEntrySignerPublicKey(prepared.key);
+            if (nativePreviousSigner) {
+                if (!nativePreviousSigner.exists) {
+                    return assertPolicyAndCommit(null);
+                }
+                if (nativePreviousSigner.publicKey) {
+                    return mapMaybePromise(this.context.assertPlainPutPolicySupported(prepared.document, undefined, nativePreviousSigner.publicKey), () => commit());
+                }
+            }
+            throw this.context.nativeModeError("requires native document context/signature facts");
+        }
+        return assertPolicyAndCommit();
+    }
+    async putMany(docs, options) {
+        if (docs.length === 0) {
+            return { entries: [], removed: [] };
+        }
+        const putOptions = this.context.normalizePutOptions(options);
+        for (const doc of docs) {
+            this.context.assertPlainPutSupported(doc, putOptions);
+        }
+        const prepared = docs.map((doc) => this.context.preparePlainPut(doc));
+        if (this.context.hasDuplicatePreparedPutKeys(prepared)) {
+            const results = [];
+            for (const doc of docs) {
+                results.push(await this.put(doc, putOptions));
+            }
+            let entries;
+            return {
+                get entries() {
+                    return (entries ??= results.map((result) => result.entry));
+                },
+                removed: results.flatMap((result) => result.removed),
+            };
+        }
+        this.context.assertPlainPutManySupported(docs, putOptions);
+        let existingContexts;
+        let previousSignerPublicKeys;
+        const policyNeedsExistingContext = this.context.plainPutPolicyNeedsExistingContext();
+        const requiredPreviousSignerPublicKey = this.context.getNativeAppendRequiredPreviousSignerPublicKey();
+        const useNativeExistingDocumentContext = putOptions?.unique !== true &&
+            (!policyNeedsExistingContext || !!requiredPreviousSignerPublicKey);
+        if (putOptions?.unique !== true && !useNativeExistingDocumentContext) {
+            const keys = prepared.map((item) => item.key);
+            const nativeContexts = this.context.getNativeIndexedContextsAndPreviousSignerPublicKeys(keys);
+            if (nativeContexts) {
+                existingContexts = nativeContexts.contexts;
+                if (policyNeedsExistingContext) {
+                    previousSignerPublicKeys = nativeContexts.publicKeys;
+                }
+            }
+            else {
+                throw this.context.nativeModeError("requires native document context/signature batch facts");
+            }
+        }
+        if (existingContexts &&
+            !previousSignerPublicKeys &&
+            policyNeedsExistingContext) {
+            const previousHeads = existingContexts.map((existing) => this.context.getIndexedContextHead(existing ?? null));
+            if (previousHeads.some((head) => head != null)) {
+                previousSignerPublicKeys = new Array(prepared.length);
+                const lookupIndexes = [];
+                const lookupHashes = [];
+                for (let i = 0; i < previousHeads.length; i++) {
+                    const head = previousHeads[i];
+                    if (head) {
+                        lookupIndexes.push(i);
+                        lookupHashes.push(head);
+                    }
+                }
+                const lookup = this.context.getNativeEntrySignerPublicKeys(lookupHashes);
+                if (lookup) {
+                    for (let i = 0; i < lookup.length; i++) {
+                        previousSignerPublicKeys[lookupIndexes[i]] = lookup[i];
+                    }
+                }
+            }
+        }
+        if (!requiredPreviousSignerPublicKey) {
+            await Promise.all(prepared.map((item, index) => this.context.assertPlainPutPolicySupported(item.document, existingContexts ? (existingContexts[index] ?? null) : undefined, previousSignerPublicKeys?.[index])));
+        }
+        return mapMaybePromise(this.context.commitNativeDocumentAppendMany({
+            puts: prepared.map((item, index) => ({
+                document: item.document,
+                key: item.key,
+                documentBytes: item.encodedDocument,
+                operationPayloadBytes: item.operationPayloadBytes,
+                unique: putOptions?.unique,
+                requiredPreviousSignerPublicKey,
+                existing: existingContexts
+                    ? (existingContexts[index] ?? null)
+                    : useNativeExistingDocumentContext
+                        ? undefined
+                        : null,
+            })),
+            resolveTrimmedEntries: this.context.shouldResolveTrimmedEntries(),
+            options: putOptions,
+            useNativeExistingDocumentContext,
+        }), (documentAppendCommit) => {
+            if (!documentAppendCommit) {
+                throw this.context.nativeModeError("requires native batched payload append support");
+            }
+            return mapMaybePromise(this.context.handlePreparedPlainPutManyCommit(documentAppendCommit), () => {
+                for (const commit of documentAppendCommit.commits) {
+                    this.context.keepEntry(commit.append.hash);
+                }
+                return {
+                    get entries() {
+                        return documentAppendCommit.entries;
+                    },
+                    removed: documentAppendCommit.removed,
+                };
+            });
+        });
+    }
+    del(id, options) {
+        const deleteOptions = this.context.normalizePutOptions(options);
+        return this.context.deleteDocument(id, deleteOptions);
+    }
+}
+const nativeDocumentContextFactsAsContext = (facts) => facts;
+const asTrustedDocumentSharedLog = (log) => log;
+const documentIndexStoreKey = (id) => {
+    const key = indexerTypes.toIdeable(id);
+    if (key instanceof Uint8Array || ArrayBuffer.isView(key)) {
+        return `bytes:${id.primitive.toString()}`;
+    }
+    return `${typeof key}:${key.toString()}`;
+};
+const asTrustedDocumentIndex = (index) => index;
 let Documents = (() => {
     let _classDecorators = [variant("documents")];
     let _classDescriptor;
@@ -86,6 +354,21 @@ let Documents = (() => {
         _index = (__runInitializers(this, _immutable_extraInitializers), __runInitializers(this, __index_initializers, void 0));
         _clazz = __runInitializers(this, __index_extraInitializers);
         _optionCanPerform;
+        _optionCanPerformNativePolicy;
+        _optionCanPerformNativeFastPath;
+        _nativeBackboneDocumentIndexEnabled = false;
+        _mode = "auto";
+        _nativeModeReplicatedOpen = false;
+        _valueClassIsProgram = false;
+        _documentChangeListeners = [];
+        _documentChangeListenerCount = 0;
+        _documentInternalChangeListenerCount = 0;
+        _documentChangeListenerTrackingInitialized = false;
+        _documentBackend;
+        _canAppendDecodedDocuments = new WeakMap();
+        _nativeDocumentIdExtractionPlan;
+        _nativeDocumentFieldExtractionPlans;
+        _hasLogTrim = false;
         idResolver;
         domain;
         strictHistory;
@@ -96,123 +379,848 @@ let Documents = (() => {
             this.log = new SharedLog(properties);
             this.immutable = properties?.immutable ?? false;
             this._index = properties?.index || new DocumentIndex();
+            this.trackDocumentChangeListeners();
+            this._documentBackend = this.createDocumentBackend();
         }
         get index() {
             return this._index;
         }
-        getLocalIndexedContext(key) {
-            return this._index.index.get(key, {
-                shape: INDEX_CONTEXT_SHAPE,
-            });
+        isNativeMode() {
+            return this._mode === "native";
         }
-        getExistingContext(existing) {
-            return isResultIndexedValue(existing)
-                ? existing.context
-                : existing?.value.__context;
+        createDocumentBackend() {
+            return this.isNativeMode()
+                ? new NativeDocumentBackend(this.createNativeDocumentBackendContext())
+                : new CompatDocumentBackend(this.putCompatDocumentBackend.bind(this), this.putManyCompatDocumentBackend.bind(this), this.delCompatDocumentBackend.bind(this));
         }
-        get changes() {
-            return this.events;
+        createNativeDocumentBackendContext() {
+            return {
+                assertPlainPutSupported: (doc, options) => {
+                    this.assertNativeModePlainPutSupported(doc, options);
+                },
+                assertPlainPutPolicySupported: (doc, existing, previousSignerPublicKey) => this.assertNativeModePlainPutPolicySupported(doc, existing, previousSignerPublicKey),
+                assertPlainPutManySupported: (docs, options) => {
+                    this.assertNativeModePlainPutManySupported(docs, options);
+                },
+                normalizePutOptions: (options) => this.normalizeNativeModePutOptions(options),
+                preparePlainPut: (doc) => this.preparePlainPut(doc),
+                hasDuplicatePreparedPutKeys: (prepared) => this.hasDuplicatePreparedPutKeys(prepared),
+                getIndexedContextHead: (existing) => this.getExistingContext(existing)?.head,
+                getNextFromIndexedContext: (existing) => {
+                    const existingHead = this.getExistingContext(existing)?.head;
+                    return existingHead
+                        ? this.nextFromIndexedContext(existingHead, existing)
+                        : undefined;
+                },
+                getNativeEntrySignerPublicKeys: (hashes) => this.getNativeEntrySignerPublicKeys(hashes),
+                getNativePreviousEntrySignerPublicKey: (key) => this.getNativePreviousEntrySignerPublicKey(key),
+                getNativeIndexedContextsAndPreviousSignerPublicKeys: (keys) => this.getNativeIndexedContextsAndPreviousSignerPublicKeys(keys),
+                getNativeAppendRequiredPreviousSignerPublicKey: () => this.nativePlainPutPolicyRequiredPreviousSignerPublicKey(),
+                plainPutPolicyNeedsExistingContext: () => this.nativePlainPutPolicyNeedsPreviousEntries(),
+                shouldResolveTrimmedEntries: () => {
+                    return !this._index.canGetIndexedKeyByHead();
+                },
+                commitNativeDocumentAppend: (input) => this.commitNativeDocumentAppend(input),
+                commitNativeDocumentAppendMany: (input) => this.commitNativeDocumentAppendMany(input),
+                handlePreparedPlainPutCommit: (commit) => this.handlePreparedPlainPutCommit(commit),
+                handlePreparedPlainPutManyCommit: (commit) => this.handlePreparedPlainPutManyCommit(commit),
+                deleteDocument: (id, options) => this.delNativeDocumentBackend(id, options),
+                keepEntry: (hash) => {
+                    this.keepCache?.add(hash);
+                },
+                nativeModeError: (message) => this.nativeModeError(message),
+            };
         }
-        async maybeSubprogramOpen(value) {
-            if (await this.canOpen(value)) {
-                return (await this.node.open(value, {
-                    parent: this,
-                    existing: "reuse",
-                })); // TODO types
+        nativeModeError(message) {
+            return new NativeDocumentModeError(`Documents native mode ${message}`);
+        }
+        assertNativeModeOpenOptions(options) {
+            if (!this.isNativeMode()) {
+                return;
+            }
+            const unsupported = [];
+            const nativeBackbone = options.nativeBackbone;
+            const indexTransform = options.index;
+            const nativeIndexTransformDescriptor = typeof indexTransform?.transform === "function"
+                ? getDocumentTransformDescriptor(indexTransform.transform)
+                : undefined;
+            if (!nativeBackbone ||
+                typeof nativeBackbone !== "object" ||
+                nativeBackbone.documentIndex !== true) {
+                unsupported.push("missing nativeBackbone.documentIndex");
+            }
+            else if (nativeBackbone.optional !== false) {
+                unsupported.push("optional nativeBackbone");
+            }
+            else if (!nativeBackbone.coordinatePersistence) {
+                unsupported.push("missing nativeBackbone.coordinatePersistence");
+            }
+            else if (nativeBackbone.heads === false) {
+                unsupported.push("disabled native heads");
+            }
+            if (options.domain) {
+                unsupported.push("custom domain");
+            }
+            if (options.compatibility != null) {
+                unsupported.push("legacy compatibility");
+            }
+            if (options.strictHistory) {
+                unsupported.push("strict history");
+            }
+            if (this.immutable) {
+                unsupported.push("immutable documents");
+            }
+            if (options.appendDurability) {
+                unsupported.push("custom append durability");
+            }
+            if (Program.isPrototypeOf(options.type)) {
+                unsupported.push("program-valued document type");
+            }
+            if (options.canPerform &&
+                !getCanPerformPolicyDescriptor(options.canPerform)) {
+                unsupported.push("arbitrary canPerform");
+            }
+            if (options.canOpen) {
+                unsupported.push("custom canOpen");
+            }
+            if (options.id) {
+                unsupported.push("custom id");
+            }
+            if (options.index?.canRead) {
+                unsupported.push("custom canRead");
+            }
+            if (options.index?.canSearch) {
+                unsupported.push("custom canSearch");
+            }
+            if (options.index?.prefetch) {
+                unsupported.push("index prefetch");
+            }
+            if (options.index?.cache?.query) {
+                unsupported.push("index query cache");
+            }
+            if (options.canReplicate) {
+                unsupported.push("custom canReplicate");
+            }
+            if (options.keep) {
+                unsupported.push("custom keep");
+            }
+            if (options.fanout) {
+                unsupported.push("fanout");
+            }
+            if (options.syncronizer) {
+                unsupported.push("custom syncronizer");
+            }
+            if (options.sync?.priority) {
+                unsupported.push("custom sync priority");
+            }
+            if (options.sync?.profile) {
+                unsupported.push("custom sync profile");
+            }
+            if (options.log?.trim &&
+                (options.log.trim.type !== "length" || options.log.trim.filter?.canTrim)) {
+                unsupported.push("unsupported log trim");
+            }
+            if (indexTransform?.transform && !nativeIndexTransformDescriptor) {
+                unsupported.push("arbitrary index transform");
+            }
+            if (indexTransform?.type &&
+                !indexTransform.transform &&
+                indexTransform.type !== options.type) {
+                unsupported.push("constructor index transform");
+            }
+            if (unsupported.length > 0) {
+                throw this.nativeModeError(`does not support ${unsupported.join(", ")}`);
             }
-            return value;
         }
-        keepCache = undefined;
-        async open(options) {
-            this._clazz = options.type;
-            this.canOpen = options.canOpen;
-            /* eslint-disable */
-            if (Program.isPrototypeOf(this._clazz)) {
-                if (!this.canOpen) {
-                    throw new Error("Document store needs to be opened with canOpen option when the document type is a Program");
+        assertNativeModeReady() {
+            if (!this.isNativeMode()) {
+                return;
+            }
+            if (!this._nativeBackboneDocumentIndexEnabled) {
+                throw this.nativeModeError("requires an attached native backbone document index");
+            }
+            if (!asTrustedDocumentIndex(this._index).canPrepareNativeBackboneDocumentIndexCommitWithAppendFacts()) {
+                throw this.nativeModeError("requires a native-compatible document index transform");
+            }
+            if (!this._nativeDocumentIdExtractionPlan) {
+                throw this.nativeModeError("requires a native-compatible document id field");
+            }
+            if (this._optionCanPerformNativePolicy) {
+                const signedByFieldPaths = canPerformPolicySignedByFieldPaths(this._optionCanPerformNativePolicy);
+                for (const path of signedByFieldPaths) {
+                    if (!this.getNativeDocumentFieldExtractionPlan(path)) {
+                        const label = Array.isArray(path) ? path.join(".") : path;
+                        throw this.nativeModeError(`requires native-compatible signedByField policy path: ${label}`);
+                    }
+                }
+                const deleteFieldPaths = canPerformPolicyDeleteFieldPaths(this._optionCanPerformNativePolicy);
+                if (deleteFieldPaths.length > 0 &&
+                    !this._index.canReadNativeIndexedFieldValues(deleteFieldPaths)) {
+                    const labels = deleteFieldPaths.map((path) => Array.isArray(path) ? path.join(".") : path);
+                    throw this.nativeModeError(`requires native index to read delete policy field${labels.length > 1 ? "s" : ""}: ${labels.join(", ")}`);
                 }
             }
-            this._optionCanPerform = options.canPerform;
-            const idProperty = options.index?.idProperty ||
-                indexerTypes.getIdProperty(this._clazz) ||
-                "id";
-            const idResolver = options.id ||
-                (typeof idProperty === "string"
-                    ? (obj) => obj[idProperty]
-                    : (obj) => indexerTypes.extractFieldValue(obj, idProperty));
-            this.idResolver = idResolver;
-            this.compatibility = options.compatibility;
-            this.strictHistory = options.strictHistory ?? false;
-            await this._index.open({
-                documentEvents: this.events,
-                log: this.log,
-                canRead: options?.index?.canRead,
-                canSearch: options.index?.canSearch,
-                documentType: this._clazz,
-                transform: options.index,
-                indexBy: idProperty,
-                compatibility: options.compatibility,
-                cache: options?.index?.cache,
-                replicate: async (query, results) => {
-                    // here we arrive for all the results we want to persist.
-                    let mergeSegments = this.domain?.canProjectToOneSegment(query);
-                    await this.log.join(results.results
-                        .flat()
-                        .map((x) => isResultIndexedValue(x) && x.entries.length > 0
-                        ? x.entries[0]
-                        : x.context.head), { replicate: { assumeSynced: true, mergeSegments } });
-                },
-                dbType: this.constructor,
-                maybeOpen: this.maybeSubprogramOpen.bind(this),
-                prefetch: options.index?.prefetch,
-                includeIndexed: options.index?.includeIndexed,
+        }
+        canPerformAllowsPlainPutFastPath(doc) {
+            return (!this._optionCanPerform || !!this._optionCanPerformNativeFastPath?.(doc));
+        }
+        nativePlainPutPolicyNeedsPreviousEntries() {
+            return (!!this._optionCanPerformNativePolicy &&
+                canPerformPolicyNeedsPreviousEntries(this._optionCanPerformNativePolicy));
+        }
+        nativePlainPutPolicyRequiredPreviousSignerPublicKey() {
+            const descriptor = this._optionCanPerformNativePolicy;
+            const policyDescriptor = descriptor?.kind === "put" ? descriptor.policy : descriptor;
+            if (policyDescriptor?.kind !== "sameSignersAsPrevious") {
+                return;
+            }
+            return this.log.log.identity.publicKey
+                .publicKey;
+        }
+        unsupportedNativePutOptions(options) {
+            const unsupported = [];
+            if (options?.canAppend) {
+                unsupported.push("per-call canAppend");
+            }
+            if (options?.onChange) {
+                unsupported.push("per-call onChange");
+            }
+            if (options?.signers) {
+                unsupported.push("custom signers");
+            }
+            if (options?.identity) {
+                unsupported.push("custom identity");
+            }
+            if (options?.encryption) {
+                unsupported.push("encryption");
+            }
+            if (options?.trim) {
+                unsupported.push("per-call trim");
+            }
+            if (options?.durability) {
+                unsupported.push("per-call durability");
+            }
+            if (options?.deferIndexWrite !== undefined) {
+                unsupported.push("per-call index write deferral");
+            }
+            if (options?.meta?.type) {
+                unsupported.push("custom entry type");
+            }
+            if (options?.meta && "data" in options.meta) {
+                unsupported.push("custom metadata");
+            }
+            if (options?.meta?.next) {
+                unsupported.push("custom next");
+            }
+            if (options?.meta?.timestamp) {
+                unsupported.push("custom timestamp");
+            }
+            if (options?.meta?.gidSeed) {
+                unsupported.push("custom gid seed");
+            }
+            if (options?.replicate === true) {
+                unsupported.push("replicated put");
+            }
+            if (options?.target && options.target !== "none") {
+                unsupported.push("non-local target");
+            }
+            if (options?.delivery !== undefined && options.delivery !== false) {
+                unsupported.push("delivery");
+            }
+            if (options?.checkRemote) {
+                unsupported.push("remote existing-head check");
+            }
+            if (options?.replicas !== undefined) {
+                unsupported.push("per-call replicas");
+            }
+            return unsupported;
+        }
+        assertNativeModePlainPutSupported(doc, options) {
+            if (!this.isNativeMode()) {
+                return false;
+            }
+            const unsupported = this.unsupportedNativePutOptions(options);
+            if (this.immutable) {
+                unsupported.push("immutable documents");
+            }
+            if (this.strictHistory) {
+                unsupported.push("strict history");
+            }
+            if (this.compatibility === 6) {
+                unsupported.push("legacy compatibility");
+            }
+            if (Program.isPrototypeOf(this._clazz)) {
+                unsupported.push("program-valued document type");
+            }
+            if (unsupported.length > 0) {
+                throw this.nativeModeError(`does not support ${unsupported.join(", ")}`);
+            }
+            return true;
+        }
+        assertNativeModePlainPutPolicySupported(doc, existing, previousSignerPublicKey) {
+            return mapMaybePromise(this.canPerformAllowsNativePlainPut(doc, existing, previousSignerPublicKey), (allowed) => {
+                if (!allowed) {
+                    throw this.nativeModeError("canPerform policy rejected this document");
+                }
             });
-            // document v6 and below need log compatibility of v8 or below
-            // document v7 needs log compatibility of v9
-            let logCompatiblity = undefined;
-            if (options.compatibility === 6) {
-                logCompatiblity = 8;
+        }
+        canPerformAllowsNativePlainPut(doc, existing, previousSignerPublicKey) {
+            if (!this._optionCanPerform) {
+                return true;
             }
-            else if (options.compatibility === 7) {
-                logCompatiblity = 9;
+            if (!this._optionCanPerformNativePolicy) {
+                return false;
             }
-            this.domain = options.domain?.(this);
-            let keepFunction;
-            if (options?.keep === "self") {
-                this.keepCache = new Set();
-                keepFunction = async (e) => {
-                    if (this.keepCache?.has(e.hash)) {
+            if (!this.nativePlainPutPolicyNeedsPreviousEntries()) {
+                return this.canPerformAllowsPlainPutFastPath(doc);
+            }
+            const previousEntries = [];
+            const existingHead = this.getExistingContext(existing)?.head;
+            if (existingHead) {
+                if (previousSignerPublicKey) {
+                    return this.nativePutPolicyAllows(this._optionCanPerformNativePolicy, doc, previousEntries, [previousSignerPublicKey]);
+                }
+                if (this.isNativeMode()) {
+                    throw this.nativeModeError("requires native previous signer facts");
+                }
+                return mapMaybePromise(this._resolveEntry(existingHead, {
+                    remote: true,
+                }), (previousEntry) => {
+                    if (previousEntry) {
+                        previousEntries.push(previousEntry);
+                    }
+                    return this.nativePutPolicyAllows(this._optionCanPerformNativePolicy, doc, previousEntries);
+                });
+            }
+            return this.nativePutPolicyAllows(this._optionCanPerformNativePolicy, doc, previousEntries);
+        }
+        async nativePutPolicyAllows(descriptor, doc, previousEntries, previousSignerPublicKeys = []) {
+            return this.nativePutOperationPolicyAllows(descriptor, undefined, doc, previousEntries, previousSignerPublicKeys);
+        }
+        nativeFieldValueMatchesLocalPublicKey(value) {
+            const localPublicKey = this.log.log.identity.publicKey;
+            return this.nativeFieldValueMatchesPublicKey(value, localPublicKey);
+        }
+        nativeFieldValueMatchesPublicKey(value, publicKey) {
+            const localRawPublicKey = publicKey.publicKey;
+            return (value instanceof Uint8Array &&
+                (bytesEqual(value, publicKey.bytes) ||
+                    (localRawPublicKey ? bytesEqual(value, localRawPublicKey) : false)));
+        }
+        nativeFieldValueMatchesPublicKeys(value, publicKeys) {
+            for (const publicKey of publicKeys) {
+                if (this.nativeFieldValueMatchesPublicKey(value, publicKey)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        nativeDeletePolicyNeedsEntryPublicKeys(descriptor) {
+            switch (descriptor.kind) {
+                case "signedByPublicKey":
+                case "deleteSignedByExistingField":
+                    return true;
+                case "delete":
+                    return this.nativeDeletePolicyNeedsEntryPublicKeys(descriptor.policy);
+                case "and":
+                case "or":
+                    return descriptor.policies.some((policy) => this.nativeDeletePolicyNeedsEntryPublicKeys(policy));
+                default:
+                    return false;
+            }
+        }
+        async nativePutOperationPolicyAllows(descriptor, operation, doc, previousEntries, previousSignerPublicKeys = [], entryPublicKeys = []) {
+            switch (descriptor.kind) {
+                case "allowAll":
+                    return createCanPerformPolicyEvaluator(descriptor, this.log.log.identity.publicKey)(doc);
+                case "signedByPublicKey":
+                    return entryPublicKeys.length > 0
+                        ? this.nativeFieldValueMatchesPublicKeys(descriptor.publicKey, entryPublicKeys)
+                        : createCanPerformPolicyEvaluator(descriptor, this.log.log.identity.publicKey)(doc);
+                case "signedByField": {
+                    if (doc) {
+                        return createCanPerformPolicyEvaluator(descriptor, this.log.log.identity.publicKey)(doc);
+                    }
+                    if (!operation) {
+                        return false;
+                    }
+                    const value = await this.getNativeDocumentFieldFromPutOperation(operation, descriptor.path);
+                    return this.nativeFieldValueMatchesPublicKeys(value, entryPublicKeys);
+                }
+                case "put":
+                    return this.nativePutOperationPolicyAllows(descriptor.policy, operation, doc, previousEntries, previousSignerPublicKeys, entryPublicKeys);
+                case "delete":
+                case "deleteSignedByExistingField":
+                    return false;
+                case "sameSignersAsPrevious": {
+                    if (previousSignerPublicKeys.length > 0) {
+                        const currentPublicKeys = entryPublicKeys.length > 0
+                            ? entryPublicKeys
+                            : [this.log.log.identity.publicKey];
+                        if (currentPublicKeys.length !== previousSignerPublicKeys.length) {
+                            return false;
+                        }
+                        for (const previousSignerPublicKey of previousSignerPublicKeys) {
+                            if (!this.nativeFieldValueMatchesPublicKeys(previousSignerPublicKey, currentPublicKeys)) {
+                                return false;
+                            }
+                        }
                         return true;
                     }
-                    let signatures = undefined;
-                    if (e instanceof Entry) {
-                        signatures = e.signatures;
+                    if (previousEntries.length === 0) {
+                        return true;
                     }
-                    else {
-                        const entry = await this.log.log.get(e.hash);
-                        signatures = entry?.signatures;
+                    const localPublicKey = this.log.log.identity.publicKey;
+                    for (const previousEntry of previousEntries) {
+                        const publicKeys = await previousEntry.getPublicKeys();
+                        if (publicKeys.length !== 1 ||
+                            !publicKeys[0].equals(localPublicKey)) {
+                            return false;
+                        }
                     }
-                    if (!signatures) {
-                        return false;
+                    return true;
+                }
+                case "and":
+                    for (const policy of descriptor.policies) {
+                        if (!(await this.nativePutOperationPolicyAllows(policy, operation, doc, previousEntries, previousSignerPublicKeys, entryPublicKeys))) {
+                            return false;
+                        }
                     }
-                    for (const signature of signatures) {
-                        if (signature.publicKey.equals(this.node.identity.publicKey)) {
-                            this.keepCache?.add(e.hash);
+                    return true;
+                case "or":
+                    for (const policy of descriptor.policies) {
+                        if (await this.nativePutOperationPolicyAllows(policy, operation, doc, previousEntries, previousSignerPublicKeys, entryPublicKeys)) {
                             return true;
                         }
                     }
-                    return false; // TODO also cache this?
-                };
-            }
-            else {
-                keepFunction = options?.keep;
+                    return false;
             }
-            await this.log.open({
-                encoding: BORSH_ENCODING_OPERATION,
-                canReplicate: options?.canReplicate,
-                canAppend: this.canAppend.bind(this),
+        }
+        async nativeDeleteOperationPolicyAllows(descriptor, operation, entryPublicKeys) {
+            switch (descriptor.kind) {
+                case "allowAll":
+                    return true;
+                case "signedByPublicKey":
+                    return entryPublicKeys
+                        ? this.nativeFieldValueMatchesPublicKeys(descriptor.publicKey, entryPublicKeys)
+                        : this.nativeFieldValueMatchesLocalPublicKey(descriptor.publicKey);
+                case "delete":
+                    return this.nativeDeleteOperationPolicyAllows(descriptor.policy, operation, entryPublicKeys);
+                case "deleteSignedByExistingField": {
+                    const value = this.getNativeDeletePolicyFieldValue(operation, descriptor.path);
+                    return entryPublicKeys
+                        ? this.nativeFieldValueMatchesPublicKeys(value, entryPublicKeys)
+                        : this.nativeFieldValueMatchesLocalPublicKey(value);
+                }
+                case "and":
+                    for (const policy of descriptor.policies) {
+                        if (!(await this.nativeDeleteOperationPolicyAllows(policy, operation, entryPublicKeys))) {
+                            return false;
+                        }
+                    }
+                    return true;
+                case "or":
+                    for (const policy of descriptor.policies) {
+                        if (await this.nativeDeleteOperationPolicyAllows(policy, operation, entryPublicKeys)) {
+                            return true;
+                        }
+                    }
+                    return false;
+                case "put":
+                case "signedByField":
+                case "sameSignersAsPrevious":
+                    return false;
+            }
+        }
+        assertNativeModePlainPutManySupported(docs, options) {
+            if (!this.isNativeMode()) {
+                return;
+            }
+            const unsupported = this.unsupportedNativePutOptions(options);
+            if (this.immutable) {
+                unsupported.push("immutable documents");
+            }
+            if (this.strictHistory) {
+                unsupported.push("strict history");
+            }
+            if (this.compatibility === 6) {
+                unsupported.push("legacy compatibility");
+            }
+            if (Program.isPrototypeOf(this._clazz)) {
+                unsupported.push("program-valued document type");
+            }
+            if (!asTrustedDocumentIndex(this._index).canUseNativeBackboneContextualBatch()) {
+                unsupported.push("native batch document index");
+            }
+            if (unsupported.length > 0) {
+                throw this.nativeModeError(`does not support ${unsupported.join(", ")}`);
+            }
+        }
+        assertNativeModeDeleteSupported(options) {
+            if (!this.isNativeMode()) {
+                return;
+            }
+            const unsupported = this.unsupportedNativePutOptions(options);
+            if (options?.unique !== undefined) {
+                unsupported.push("unique delete");
+            }
+            if (this.immutable) {
+                unsupported.push("immutable documents");
+            }
+            if (this.strictHistory) {
+                unsupported.push("strict history");
+            }
+            if (this.compatibility === 6) {
+                unsupported.push("legacy compatibility");
+            }
+            if (Program.isPrototypeOf(this._clazz)) {
+                unsupported.push("program-valued document type");
+            }
+            if (unsupported.length > 0) {
+                throw this.nativeModeError(`does not support ${unsupported.join(", ")}`);
+            }
+        }
+        async canPerformAllowsNativeDelete(properties) {
+            if (!this._optionCanPerform) {
+                return true;
+            }
+            if (!this._optionCanPerformNativePolicy) {
+                return false;
+            }
+            if (this.isNativeMode()) {
+                return this.nativeDeleteOperationPolicyAllows(this._optionCanPerformNativePolicy, properties.operation);
+            }
+            let deleteValue;
+            if (canPerformPolicyNeedsDeleteValue(this._optionCanPerformNativePolicy)) {
+                deleteValue = await properties.getExistingDocument?.();
+                if (deleteValue === undefined) {
+                    const existingEntry = await properties.getExistingEntry();
+                    const existingOperation = await existingEntry.getPayloadValue();
+                    if (isPutOperation(existingOperation)) {
+                        deleteValue = this._index.valueEncoding.decoder(existingOperation.data);
+                    }
+                }
+            }
+            return createCanPerformDeletePolicyEvaluator(this._optionCanPerformNativePolicy, this.log.log.identity.publicKey)(deleteValue);
+        }
+        normalizeNativeModePutOptions(options) {
+            if (!this.isNativeMode()) {
+                return options;
+            }
+            if (options?.replicate === false && options.target === "none") {
+                return options;
+            }
+            if (this._nativeModeReplicatedOpen) {
+                return options?.target === "none"
+                    ? options
+                    : {
+                        ...options,
+                        target: "none",
+                    };
+            }
+            const cached = cachedNativeLocalPutOptions(options);
+            if (cached) {
+                return cached;
+            }
+            return {
+                ...options,
+                replicate: false,
+                target: "none",
+            };
+        }
+        trackDocumentChangeListeners() {
+            if (this._documentChangeListenerTrackingInitialized) {
+                return;
+            }
+            this._documentChangeListenerTrackingInitialized = true;
+            this._documentChangeListeners ??= [];
+            this._documentChangeListenerCount ??= 0;
+            this._documentInternalChangeListenerCount ??= 0;
+            const events = this.events;
+            const addEventListener = events.addEventListener.bind(events);
+            const removeEventListener = events.removeEventListener.bind(events);
+            const captureFromOptions = (options) => typeof options === "boolean"
+                ? options
+                : !!options?.capture;
+            events.addEventListener = ((type, ...args) => {
+                if (type === "change") {
+                    const listener = args[0];
+                    const capture = captureFromOptions(args[1]);
+                    if (listener &&
+                        !this._documentChangeListeners.some((entry) => entry.listener === listener && entry.capture === capture)) {
+                        this._documentChangeListeners.push({ listener, capture });
+                        this._documentChangeListenerCount =
+                            this._documentChangeListeners.length;
+                    }
+                }
+                return addEventListener(type, ...args);
+            });
+            events.removeEventListener = ((type, ...args) => {
+                if (type === "change") {
+                    const listener = args[0];
+                    const capture = captureFromOptions(args[1]);
+                    const index = this._documentChangeListeners.findIndex((entry) => entry.listener === listener && entry.capture === capture);
+                    if (index >= 0) {
+                        this._documentChangeListeners.splice(index, 1);
+                        this._documentChangeListenerCount =
+                            this._documentChangeListeners.length;
+                    }
+                }
+                return removeEventListener(type, ...args);
+            });
+        }
+        getLocalIndexedContext(key) {
+            const getContextById = this._index.index.getContextById;
+            const context = getContextById?.call(this._index.index, key);
+            if (context) {
+                return Promise.resolve({
+                    id: key,
+                    value: { __context: context },
+                });
+            }
+            return this._index.index.get(key, {
+                shape: INDEX_CONTEXT_SHAPE,
+            });
+        }
+        getNativeEntrySignerPublicKeys(hashes) {
+            if (hashes.length === 0) {
+                return [];
+            }
+            const nativeGraph = this.log.log.entryIndex.properties.nativeGraph
+                ?.graph;
+            return nativeGraph?.entrySignaturePublicKeysBatch?.(hashes);
+        }
+        getSharedLogNativeBackbone() {
+            return this.log._nativeBackbone;
+        }
+        getNativePreviousEntrySignerPublicKey(key) {
+            const nativeBackbone = this.getSharedLogNativeBackbone();
+            return nativeBackbone?.documentPreviousSignaturePublicKey?.(documentIndexStoreKey(key));
+        }
+        async getNativePreviousEntrySignerPublicKeyForPutOperation(operation) {
+            const keyValue = await this.getNativeDocumentIdFromPutOperation(operation);
+            return keyValue == null
+                ? undefined
+                : this.getNativePreviousEntrySignerPublicKey(indexerTypes.toId(keyValue));
+        }
+        getNativeIndexedContext(key) {
+            const nativeBackbone = this.getSharedLogNativeBackbone();
+            const row = nativeBackbone?.documentContext?.(documentIndexStoreKey(key));
+            const context = row
+                ? {
+                    created: BigInt(row[0]),
+                    modified: BigInt(row[1]),
+                    head: row[2],
+                    gid: row[3],
+                    size: row[4],
+                }
+                : undefined;
+            return context
+                ? {
+                    id: key,
+                    value: {
+                        __context: nativeDocumentContextFactsAsContext(context),
+                    },
+                }
+                : undefined;
+        }
+        getNativeModeIndexedContext(key) {
+            if (!this.hasNativeDocumentContextLookup()) {
+                throw this.nativeModeError("requires native document context lookup");
+            }
+            return this.getNativeIndexedContext(key);
+        }
+        hasNativeDocumentContextLookup() {
+            const nativeBackbone = this.getSharedLogNativeBackbone();
+            return typeof nativeBackbone?.documentContext === "function";
+        }
+        getNativeIndexedContextsAndPreviousSignerPublicKeys(keys) {
+            if (keys.length === 0) {
+                return { contexts: [], publicKeys: [] };
+            }
+            const nativeBackbone = this.getSharedLogNativeBackbone();
+            const rows = nativeBackbone?.documentContextsAndPreviousSignaturePublicKeys?.(keys.map(documentIndexStoreKey));
+            if (!rows) {
+                return;
+            }
+            return {
+                contexts: rows.map((row, index) => row.context
+                    ? {
+                        id: keys[index],
+                        value: {
+                            __context: nativeDocumentContextFactsAsContext(row.context),
+                        },
+                    }
+                    : undefined),
+                publicKeys: rows.map((row) => row.publicKey),
+            };
+        }
+        getExistingContext(existing) {
+            return isResultIndexedValue(existing)
+                ? existing.context
+                : existing?.value.__context;
+        }
+        documentFromIdentityIndexedValue(indexed) {
+            const value = indexed?.value;
+            if (!value) {
+                return;
+            }
+            if (typeof value !== "object") {
+                return value;
+            }
+            const document = Object.assign(Object.create(this._clazz.prototype), value);
+            delete document.__context;
+            return document;
+        }
+        async getLocalIdentityDocumentByHead(head) {
+            return this.documentFromIdentityIndexedValue(await this._index.getIdentityIndexedByHead(head));
+        }
+        async getLocalIndexedDocumentForNativeDeletePolicy(key) {
+            if (!this._optionCanPerformNativePolicy) {
+                return;
+            }
+            const fieldPaths = canPerformPolicyDeleteFieldPaths(this._optionCanPerformNativePolicy);
+            if (fieldPaths.length === 0 ||
+                !this._index.canReadOriginalFieldPathsFromIndexedValue(fieldPaths)) {
+                return;
+            }
+            return (await this._index.get(key, {
+                local: true,
+                remote: false,
+                resolve: false,
+            }));
+        }
+        getNativeDeletePolicyFieldValue(operation, path) {
+            const key = operation.key instanceof indexerTypes.IdKey
+                ? operation.key
+                : indexerTypes.toId(operation.key);
+            return this._index.getNativeIndexedFieldValue(key, path);
+        }
+        get changes() {
+            return this.events;
+        }
+        async maybeSubprogramOpen(value) {
+            if (await this.canOpen(value)) {
+                return (await this.node.open(value, {
+                    parent: this,
+                    existing: "reuse",
+                }));
+            }
+            return value;
+        }
+        keepCache = undefined;
+        async open(options) {
+            this.trackDocumentChangeListeners();
+            // Deserialized instances skip constructor/field initializers (borsh creates
+            // objects via Object.create), so re-establish constructor-only state here.
+            this._canAppendDecodedDocuments ??= new WeakMap();
+            this._clazz = options.type;
+            this._valueClassIsProgram = Program.isPrototypeOf(this._clazz);
+            this.canOpen = options.canOpen;
+            this._mode = options.mode ?? "auto";
+            this._nativeModeReplicatedOpen =
+                this.isNativeMode() &&
+                    options.replicate !== undefined &&
+                    options.replicate !== false;
+            this.assertNativeModeOpenOptions(options);
+            if (Program.isPrototypeOf(this._clazz)) {
+                if (!this.canOpen) {
+                    throw new Error("Document store needs to be opened with canOpen option when the document type is a Program");
+                }
+            }
+            this._optionCanPerform = options.canPerform;
+            this._optionCanPerformNativePolicy = getCanPerformPolicyDescriptor(options.canPerform);
+            const idProperty = options.index?.idProperty ||
+                indexerTypes.getIdProperty(this._clazz) ||
+                "id";
+            const idResolver = options.id ||
+                (typeof idProperty === "string"
+                    ? (obj) => obj[idProperty]
+                    : (obj) => indexerTypes.extractFieldValue(obj, idProperty));
+            this.idResolver = idResolver;
+            this.compatibility = options.compatibility;
+            this.strictHistory = options.strictHistory ?? false;
+            this._hasLogTrim = options.log?.trim != null;
+            const changeListenersBeforeIndexOpen = this._documentChangeListenerCount;
+            await this._index.open({
+                documentEvents: this.events,
+                log: this.log,
+                canRead: options?.index?.canRead,
+                canSearch: options.index?.canSearch,
+                documentType: this._clazz,
+                transform: options.index,
+                indexBy: idProperty,
+                compatibility: options.compatibility,
+                cache: options?.index?.cache,
+                replicate: async (query, results) => {
+                    // here we arrive for all the results we want to persist.
+                    let mergeSegments = this.domain?.canProjectToOneSegment(query);
+                    await this.log.join(results.results
+                        .flat()
+                        .map((x) => isResultIndexedValue(x) && x.entries.length > 0
+                        ? x.entries[0]
+                        : x.context.head), { replicate: { assumeSynced: true, mergeSegments } });
+                },
+                dbType: this.constructor,
+                maybeOpen: this.maybeSubprogramOpen.bind(this),
+                prefetch: options.index?.prefetch,
+                includeIndexed: options.index?.includeIndexed,
+            });
+            this._documentInternalChangeListenerCount = Math.max(0, this._documentChangeListenerCount - changeListenersBeforeIndexOpen);
+            this._nativeDocumentFieldExtractionPlans ??= new Map();
+            this._nativeDocumentFieldExtractionPlans.clear();
+            this._nativeDocumentIdExtractionPlan =
+                asTrustedDocumentIndex(this._index).getNativeDocumentFieldExtractionPlan(idProperty);
+            // document v6 and below need log compatibility of v8 or below
+            // document v7 needs log compatibility of v9
+            let logCompatiblity = undefined;
+            if (options.compatibility === 6) {
+                logCompatiblity = 8;
+            }
+            else if (options.compatibility === 7) {
+                logCompatiblity = 9;
+            }
+            this.domain = options.domain?.(this);
+            let keepFunction;
+            if (options?.keep === "self") {
+                this.keepCache = new Set();
+                keepFunction = async (e) => {
+                    if (this.keepCache?.has(e.hash)) {
+                        return true;
+                    }
+                    let signatures = undefined;
+                    if (e instanceof Entry) {
+                        signatures = e.signatures;
+                    }
+                    else {
+                        const entry = await this.log.log.get(e.hash);
+                        signatures = entry?.signatures;
+                    }
+                    if (!signatures) {
+                        return false;
+                    }
+                    for (const signature of signatures) {
+                        if (signature.publicKey.equals(this.node.identity.publicKey)) {
+                            this.keepCache?.add(e.hash);
+                            return true;
+                        }
+                    }
+                    return false;
+                };
+            }
+            else {
+                keepFunction = options?.keep;
+            }
+            await this.log.open({
+                encoding: BORSH_ENCODING_OPERATION,
+                canReplicate: options?.canReplicate,
+                canAppend: this.canAppend.bind(this),
                 onChange: this.handleChanges.bind(this),
                 trim: options?.log?.trim,
+                appendDurability: options?.appendDurability,
+                nativeBackbone: options?.nativeBackbone,
+                nativeGraph: options?.nativeGraph,
+                nativeRangePlanner: options?.nativeRangePlanner,
                 replicate: options?.replicate,
                 replicas: options?.replicas,
                 respondToIHaveTimeout: options?.respondToIHaveTimeout,
@@ -223,14 +1231,30 @@ let Documents = (() => {
                 waitForPruneDelay: options?.waitForPruneDelay,
                 distributionDebounceTime: options?.distributionDebounceTime,
                 strictFullReplicaFallback: false,
-                domain: (options?.domain
-                    ? (log) => options.domain(this)
-                    : undefined), /// TODO types,
+                domain: options?.domain
+                    ? () => options.domain(this)
+                    : undefined,
                 compatibility: logCompatiblity,
                 eagerBlocks: options?.eagerBlocks,
                 fanout: options?.fanout,
                 keep: keepFunction,
             });
+            this._nativeBackboneDocumentIndexEnabled = false;
+            if (this._mode !== "compat" &&
+                options?.nativeBackbone &&
+                typeof options.nativeBackbone !== "boolean" &&
+                options.nativeBackbone.documentIndex === true) {
+                this._nativeBackboneDocumentIndexEnabled =
+                    asTrustedDocumentIndex(this._index).attachNativeBackboneDocumentIndex(this.getSharedLogNativeBackbone(), { preserveExisting: this._mode === "native" }) === true;
+                if (this._nativeBackboneDocumentIndexEnabled) {
+                    await initializeDocumentRust();
+                }
+            }
+            this._optionCanPerformNativeFastPath = this._optionCanPerformNativePolicy
+                ? createCanPerformPolicyEvaluator(this._optionCanPerformNativePolicy, this.log.log.identity.publicKey)
+                : undefined;
+            this.assertNativeModeReady();
+            this._documentBackend = this.createDocumentBackend();
         }
         async recover() {
             return this.log.recover();
@@ -248,229 +1272,1795 @@ let Documents = (() => {
             }
             try {
                 let operation = l0;
-                let document = reference?.document;
-                if (!document) {
-                    if (isPutOperation(l0)) {
-                        document = this._index.valueEncoding.decoder(l0.data);
-                        if (!document) {
-                            return false;
-                        }
-                    }
-                    else if (isDeleteOperation(l0)) {
-                        // Nothing to do here by default
-                        // checking if the document exists is not necessary
-                        // since it might already be deleted
+                if (this._optionCanPerform) {
+                    if (this._optionCanPerformNativePolicy && this.isNativeMode()) {
+                        return this.nativeCanPerformAllowsAppend(this._optionCanPerformNativePolicy, operation, entry, reference?.document);
                     }
-                    else {
-                        throw new Error("Unsupported operation");
+                    let document = reference?.document;
+                    if (!document) {
+                        if (isPutOperation(l0)) {
+                            document =
+                                this._canAppendDecodedDocuments.get(l0) ??
+                                    this._index.valueEncoding.decoder(l0.data);
+                            if (!document) {
+                                return false;
+                            }
+                        }
+                        else if (isDeleteOperation(l0)) {
+                            // Nothing to do here by default.
+                            // Checking if the document exists is not necessary since it
+                            // might already be deleted.
+                        }
+                        else {
+                            throw new Error("Unsupported operation");
+                        }
                     }
-                }
-                if (this._optionCanPerform) {
+                    const previousEntries = this._optionCanPerformNativePolicy &&
+                        isPutOperation(operation) &&
+                        canPerformPolicyNeedsPreviousEntries(this._optionCanPerformNativePolicy)
+                        ? await this.resolveCanPerformPreviousEntries(entry)
+                        : undefined;
+                    const deleteValue = this._optionCanPerformNativePolicy &&
+                        isDeleteOperation(operation) &&
+                        canPerformPolicyNeedsDeleteValue(this._optionCanPerformNativePolicy)
+                        ? await this.resolveCanPerformDeleteValue(operation)
+                        : undefined;
                     if (!(await this._optionCanPerform(isPutOperation(operation)
                         ? {
                             type: "put",
                             value: document,
                             operation,
                             entry: entry,
+                            previousEntries,
                         }
                         : {
                             type: "delete",
+                            value: deleteValue,
                             operation,
                             entry: entry,
                         }))) {
                         return false;
                     }
                 }
-            }
-            catch (error) {
-                if (error instanceof BorshError) {
-                    warn("Received payload that could not be decoded, skipping");
-                    return false;
+            }
+            catch (error) {
+                if (error instanceof BorshError) {
+                    warn("Received payload that could not be decoded, skipping");
+                    return false;
+                }
+                throw error;
+            }
+            return true;
+        }
+        async nativeCanPerformAllowsAppend(descriptor, operation, entry, document) {
+            if (isPutOperation(operation)) {
+                let previousSignerPublicKeys = [];
+                let previousEntries = [];
+                if (canPerformPolicyNeedsPreviousEntries(descriptor)) {
+                    const lookup = this.getNativeEntrySignerPublicKeys(entry.meta.next);
+                    if (lookup && lookup.every((key) => key != null)) {
+                        previousSignerPublicKeys = lookup;
+                    }
+                    else if (this.isNativeMode()) {
+                        if (entry.meta.next.length > 1) {
+                            return false;
+                        }
+                        const previousSigner = await this.getNativePreviousEntrySignerPublicKeyForPutOperation(operation);
+                        if (previousSigner?.publicKey) {
+                            previousSignerPublicKeys = [previousSigner.publicKey];
+                        }
+                        else if (previousSigner?.exists || entry.meta.next.length > 0) {
+                            return false;
+                        }
+                    }
+                    else {
+                        previousEntries = await this.resolveCanPerformPreviousEntries(entry);
+                    }
+                }
+                const entryPublicKeys = !document && canPerformPolicyPutNeedsEntryPublicKeys(descriptor)
+                    ? entry.publicKeys.length > 0
+                        ? entry.publicKeys
+                        : await entry.getPublicKeys()
+                    : [];
+                return this.nativePutOperationPolicyAllows(descriptor, operation, document, previousEntries, previousSignerPublicKeys, entryPublicKeys);
+            }
+            const entryPublicKeys = this.nativeDeletePolicyNeedsEntryPublicKeys(descriptor)
+                ? entry.publicKeys.length > 0
+                    ? entry.publicKeys
+                    : await entry.getPublicKeys()
+                : undefined;
+            return this.nativeDeleteOperationPolicyAllows(descriptor, operation, entryPublicKeys);
+        }
+        async resolveCanPerformPreviousEntries(entry) {
+            const entries = [];
+            for (const hash of entry.meta.next) {
+                const previous = await this._resolveEntry(hash);
+                if (previous) {
+                    entries.push(previous);
+                }
+            }
+            return entries;
+        }
+        async resolveCanPerformDeleteValue(operation, options) {
+            const key = operation.key instanceof indexerTypes.IdKey
+                ? operation.key
+                : indexerTypes.toId(operation.key);
+            const existing = await this.getLocalIndexedContext(key);
+            const existingHead = this.getExistingContext(existing)?.head;
+            if (!existingHead) {
+                return;
+            }
+            const indexedDocument = await this.getLocalIdentityDocumentByHead(existingHead);
+            if (indexedDocument) {
+                return indexedDocument;
+            }
+            const indexedPolicyDocument = await this.getLocalIndexedDocumentForNativeDeletePolicy(key);
+            if (indexedPolicyDocument) {
+                return indexedPolicyDocument;
+            }
+            if (options?.allowEntryFallback === false) {
+                return;
+            }
+            const existingEntry = await this._resolveEntry(existingHead, {
+                remote: true,
+            });
+            const existingOperation = await existingEntry.getPayloadValue();
+            if (!isPutOperation(existingOperation)) {
+                return;
+            }
+            return this._index.valueEncoding.decoder(existingOperation.data);
+        }
+        async _canAppend(entry, reference) {
+            const resolve = async (history) => {
+                return typeof history === "string"
+                    ? this.log.log.get(history) ||
+                        (await Entry.fromMultihash(this.log.log.blocks, history))
+                    : history;
+            };
+            const pointsToHistory = async (history) => {
+                // make sure nexts only points to this document at some point in history
+                let current = await resolve(history);
+                const next = entry.meta.next[0];
+                while (current?.hash &&
+                    next !== current?.hash &&
+                    current.meta.next.length > 0) {
+                    current = await this.log.log.get(current.meta.next[0]);
+                }
+                if (current?.hash === next) {
+                    return true; // Ok, we are pointing this new edit to some exising point in time of the old document
+                }
+                return false;
+            };
+            let initialized = false;
+            const ensureInitialized = () => {
+                if (!initialized) {
+                    entry.init({
+                        encoding: this.log.log.encoding,
+                        keychain: this.node.services.keychain,
+                    });
+                    initialized = true;
+                }
+            };
+            try {
+                const operation = reference?.operation ||
+                    (await this.getAppendOperation(entry, ensureInitialized));
+                if (!operation) {
+                    return false;
+                }
+                if (isPutOperation(operation)) {
+                    // check nexts
+                    const putOperation = operation;
+                    let keyValue;
+                    if (reference?.document) {
+                        keyValue = this.idResolver(reference.document);
+                    }
+                    else {
+                        keyValue = await this.getNativeDocumentIdFromPutOperation(putOperation);
+                        if (keyValue == null) {
+                            if (this.isNativeMode()) {
+                                return false;
+                            }
+                            const value = this.index.valueEncoding.decoder(putOperation.data);
+                            this._canAppendDecodedDocuments.set(putOperation, value);
+                            keyValue = this.idResolver(value);
+                        }
+                    }
+                    const key = indexerTypes.toId(keyValue);
+                    const existingDocument = this.isNativeMode()
+                        ? this.hasNativeDocumentContextLookup()
+                            ? this.getNativeIndexedContext(key)
+                            : undefined
+                        : this.immutable
+                            ? (await this.index.getDetailed(key, {
+                                resolve: false,
+                                local: true,
+                                remote: { strategy: "fallback" },
+                            }))?.[0]?.results[0]
+                            : await this.getLocalIndexedContext(key);
+                    if (this.isNativeMode() && !this.hasNativeDocumentContextLookup()) {
+                        return false;
+                    }
+                    const existingContext = this.getExistingContext(existingDocument);
+                    if (existingContext && existingContext.head !== entry.hash) {
+                        // This can happen if we reset the operation log without resetting the index, for example during recover.
+                        if (this.immutable) {
+                            // key already exist but pick the oldest entry
+                            // this is because we can not overwrite same id if immutable
+                            if (existingContext.created < entry.meta.clock.timestamp.wallTime) {
+                                return false;
+                            }
+                            if (entry.meta.next.length > 0) {
+                                return false; // can not append to immutable document
+                            }
+                            return putOperation;
+                        }
+                        else {
+                            if (this.strictHistory) {
+                                // make sure that the next pointer exist and points to the existing documents
+                                if (entry.meta.next.length !== 1) {
+                                    return false;
+                                }
+                                if (entry.meta.next[0] === existingContext.head) {
+                                    return putOperation;
+                                }
+                                const prevEntry = await this.log.log.entryIndex.get(existingContext.head);
+                                if (!prevEntry) {
+                                    logger.error("Failed to find previous entry for document edit: " +
+                                        entry.hash);
+                                    return false;
+                                }
+                                const referenceHistoryCorrectly = await pointsToHistory(prevEntry);
+                                return referenceHistoryCorrectly ? putOperation : false;
+                            }
+                            else {
+                                return putOperation;
+                            }
+                        }
+                    }
+                    else {
+                        // Keep existing behavior: next pointers may express document dependencies.
+                    }
+                }
+                else if (isDeleteOperation(operation)) {
+                    if (entry.meta.next.length !== 1) {
+                        return false;
+                    }
+                    const deleteKey = operation.key instanceof indexerTypes.IdKey
+                        ? operation.key
+                        : indexerTypes.toId(operation.key);
+                    const existingDocument = this.isNativeMode()
+                        ? this.hasNativeDocumentContextLookup()
+                            ? this.getNativeIndexedContext(deleteKey)
+                            : undefined
+                        : this.immutable
+                            ? (await this.index.getDetailed(operation.key, {
+                                resolve: false,
+                                local: true,
+                                remote: true,
+                            }))?.[0]?.results[0]
+                            : await this.getLocalIndexedContext(deleteKey);
+                    if (this.isNativeMode() && !this.hasNativeDocumentContextLookup()) {
+                        return false;
+                    }
+                    const existingHead = this.getExistingContext(existingDocument)?.head;
+                    if (!existingHead) {
+                        // already deleted
+                        return coerceDeleteOperation(operation); // assume ok
+                    }
+                    if (entry.meta.next[0] === existingHead) {
+                        return coerceDeleteOperation(operation);
+                    }
+                    if (this.isNativeMode()) {
+                        return false;
+                    }
+                    let doc = await this.log.log.get(existingHead);
+                    if (!doc) {
+                        logger.error("Failed to find Document from head");
+                        return false;
+                    }
+                    if (await pointsToHistory(doc)) {
+                        // references the existing document
+                        return coerceDeleteOperation(operation);
+                    }
+                    return false;
+                }
+                else {
+                    throw new Error("Unsupported operation");
+                }
+                return operation;
+            }
+            catch (error) {
+                if (error instanceof AccessError) {
+                    return false; // we cant index because we can not decrypt
+                }
+                else if (error instanceof BorshError) {
+                    warn("Received payload that could not be decoded, skipping");
+                    return false;
+                }
+                throw error;
+            }
+        }
+        async getAppendOperation(entry, ensureInitialized) {
+            if (this.isNativeMode()) {
+                const operation = await this.getPlainEntryOperationFromStorage(entry);
+                if (operation) {
+                    return operation;
+                }
+                return;
+            }
+            ensureInitialized?.();
+            return entry.getPayloadValue();
+        }
+        async getPlainEntryOperationFromStorage(entry) {
+            let storageBytes;
+            try {
+                storageBytes =
+                    Entry.getPreparedStorageBytes(entry) ?? entry.getStorageBytes();
+            }
+            catch {
+                return;
+            }
+            try {
+                const payloadData = await entryV0PlainPayloadDataFromStorage(storageBytes);
+                return payloadData
+                    ? BORSH_ENCODING_OPERATION.decoder(payloadData)
+                    : undefined;
+            }
+            catch {
+                try {
+                    const payloadData = entry
+                        .payload?.data;
+                    return payloadData
+                        ? BORSH_ENCODING_OPERATION.decoder(payloadData)
+                        : undefined;
+                }
+                catch {
+                    return;
+                }
+            }
+        }
+        getNativeDocumentFieldExtractionPlan(path) {
+            const key = JSON.stringify(typeof path === "string" ? [path] : path);
+            const plans = (this._nativeDocumentFieldExtractionPlans ??= new Map());
+            if (plans.has(key)) {
+                return plans.get(key);
+            }
+            const plan = asTrustedDocumentIndex(this._index).getNativeDocumentFieldExtractionPlan(path);
+            plans.set(key, plan);
+            return plan;
+        }
+        async getNativeDocumentFieldFromPutOperation(operation, path) {
+            if (!this.isNativeMode()) {
+                return;
+            }
+            const plan = this.getNativeDocumentFieldExtractionPlan(path);
+            if (!plan) {
+                return;
+            }
+            try {
+                return await extractDocumentFieldSimple(operation.data, plan);
+            }
+            catch {
+                return;
+            }
+        }
+        async getNativeDocumentIdFromPutOperation(operation) {
+            if (!this.isNativeMode() || !this._nativeDocumentIdExtractionPlan) {
+                return;
+            }
+            try {
+                const id = await extractDocumentFieldSimple(operation.data, this._nativeDocumentIdExtractionPlan);
+                return id;
+            }
+            catch {
+                return;
+            }
+        }
+        preparePut(doc) {
+            const keyValue = this.idResolver(doc);
+            indexerTypes.checkId(keyValue);
+            let encodedDocument = serialize(doc);
+            if (encodedDocument.length > MAX_BATCH_SIZE) {
+                throw new Error(`Document is too large (${encodedDocument.length * 1e-6}) mb). Needs to be less than ${MAX_BATCH_SIZE * 1e-6} mb`);
+            }
+            const key = indexerTypes.toId(keyValue);
+            let operation;
+            let encodedOperation;
+            if (this.compatibility === 6) {
+                if (typeof keyValue === "string") {
+                    operation = new PutWithKeyOperation({
+                        key: keyValue,
+                        data: encodedDocument,
+                    });
+                }
+                else {
+                    throw new Error("Key must be a string in compatibility mode v6");
+                }
+            }
+            else {
+                encodedOperation = encodePutOperationPayload(encodedDocument);
+                encodedDocument = encodedOperation.subarray(PUT_OPERATION_PREFIX_LENGTH);
+                operation = new PutOperation({
+                    data: encodedDocument,
+                });
+            }
+            return {
+                document: doc,
+                encodedDocument,
+                encodedOperation,
+                keyValue,
+                key,
+                operation,
+            };
+        }
+        preparePlainPut(doc) {
+            if (this.compatibility === 6) {
+                throw new Error("Plain put preparation is not supported in v6 mode");
+            }
+            const keyValue = this.idResolver(doc);
+            indexerTypes.checkId(keyValue);
+            const documentBytes = serialize(doc);
+            if (documentBytes.length > MAX_BATCH_SIZE) {
+                throw new Error(`Document is too large (${documentBytes.length * 1e-6}) mb). Needs to be less than ${MAX_BATCH_SIZE * 1e-6} mb`);
+            }
+            const operationPayloadBytes = encodePutOperationPayload(documentBytes);
+            return {
+                document: doc,
+                encodedDocument: operationPayloadBytes.subarray(PUT_OPERATION_PREFIX_LENGTH),
+                operationPayloadBytes,
+                keyValue,
+                key: indexerTypes.toId(keyValue),
+            };
+        }
+        async put(doc, options) {
+            return this._documentBackend.put(doc, options);
+        }
+        async putCompatDocumentBackend(doc, options) {
+            const putOptions = this.normalizeNativeModePutOptions(options);
+            const prepared = this.canUsePlainPutFastPath(doc, putOptions)
+                ? this.preparePlainPut(doc)
+                : this.preparePut(doc);
+            let existingLocalContext;
+            let existingHead;
+            if (!putOptions?.unique) {
+                if (putOptions?.checkRemote) {
+                    existingHead = (await this._index.getDetailed(prepared.key, {
+                        resolve: false,
+                        local: true,
+                        remote: { replicate: putOptions?.replicate },
+                    }))?.[0]?.results[0]?.context.head;
+                }
+                else {
+                    existingLocalContext =
+                        (await this.getLocalIndexedContext(prepared.key)) || null;
+                    existingHead = existingLocalContext?.value.__context.head;
+                }
+            }
+            const plainPutPlan = await this.createPlainPutCommitPlan(prepared, existingHead, existingLocalContext, putOptions);
+            if (plainPutPlan) {
+                return this.commitPlainPutPlan(plainPutPlan, putOptions);
+            }
+            const operation = "operation" in prepared
+                ? prepared.operation
+                : new PutOperation({ data: prepared.encodedDocument });
+            const appended = await this.log.append(operation, {
+                ...putOptions,
+                meta: {
+                    next: existingHead ? [await this._resolveEntry(existingHead)] : [],
+                    ...putOptions?.meta,
+                },
+                canAppend: (entry) => {
+                    return this.canAppend(entry, {
+                        document: prepared.document,
+                        operation,
+                    });
+                },
+                onChange: (change) => {
+                    return this.handleChanges(change, {
+                        document: prepared.document,
+                        operation,
+                        key: prepared.key,
+                        unique: putOptions?.unique,
+                        existing: existingLocalContext,
+                    });
+                },
+                replicate: putOptions?.replicate,
+            });
+            this.keepCache?.add(appended.entry.hash);
+            return appended;
+        }
+        async putMany(docs, options) {
+            return this._documentBackend.putMany(docs, options);
+        }
+        async putManyCompatDocumentBackend(docs, options) {
+            if (docs.length === 0) {
+                return { entries: [], removed: [] };
+            }
+            if (!this.canUsePlainPutManyFastPath(docs, options)) {
+                return this.putManySequential(docs, options);
+            }
+            const prepared = docs.map((doc) => this.preparePlainPut(doc));
+            if (this.hasDuplicatePreparedPutKeys(prepared)) {
+                return this.putManySequential(docs, options);
+            }
+            const documentAppendCommit = await this.commitNativeDocumentAppendMany({
+                puts: prepared.map((item) => ({
+                    document: item.document,
+                    key: item.key,
+                    documentBytes: item.encodedDocument,
+                    operationPayloadBytes: item.operationPayloadBytes,
+                    unique: options?.unique,
+                    existing: null,
+                })),
+                resolveTrimmedEntries: !this._index.canGetIdentityIndexedByHead(),
+                options,
+            });
+            if (!documentAppendCommit) {
+                return this.putManySequential(docs, options);
+            }
+            await this.handlePreparedPlainPutManyCommit(documentAppendCommit);
+            for (const commit of documentAppendCommit.commits) {
+                this.keepCache?.add(commit.append.hash);
+            }
+            return {
+                get entries() {
+                    return documentAppendCommit.entries;
+                },
+                removed: documentAppendCommit.removed,
+            };
+        }
+        async putManySequential(docs, options) {
+            const entries = [];
+            const removed = [];
+            for (const doc of docs) {
+                const appended = await this.put(doc, options);
+                entries.push(appended.entry);
+                removed.push(...appended.removed);
+            }
+            return { entries, removed };
+        }
+        hasDuplicatePreparedPutKeys(prepared) {
+            const keys = new Set();
+            for (const item of prepared) {
+                if (keys.has(item.key.primitive)) {
+                    return true;
+                }
+                keys.add(item.key.primitive);
+            }
+            return false;
+        }
+        canUsePlainPutFastPath(doc, options) {
+            return (this._mode !== "compat" &&
+                this.canPerformAllowsPlainPutFastPath(doc) &&
+                !this.immutable &&
+                !this.strictHistory &&
+                this.compatibility !== 6 &&
+                !Program.isPrototypeOf(this._clazz) &&
+                !options?.canAppend &&
+                !options?.onChange &&
+                !options?.signers &&
+                !options?.identity &&
+                !options?.encryption &&
+                !options?.trim &&
+                !options?.durability &&
+                options?.deferIndexWrite === undefined &&
+                !options?.meta?.type &&
+                !(options?.meta && "data" in options.meta) &&
+                !options?.meta?.next &&
+                !options?.meta?.timestamp &&
+                !options?.meta?.gidSeed &&
+                options?.replicate !== true &&
+                (!options?.target || options.target === "none") &&
+                (options?.delivery === undefined || options.delivery === false) &&
+                !options?.checkRemote &&
+                options?.replicas === undefined);
+        }
+        canUsePlainPutManyFastPath(docs, options) {
+            return (options?.unique === true &&
+                options?.replicate !== true &&
+                options?.target === "none" &&
+                (options?.delivery === undefined || options.delivery === false) &&
+                docs.every((doc) => this.canUsePlainPutFastPath(doc, options)));
+        }
+        async createPlainPutCommitPlan(prepared, existingHead, existingLocalContext, options, assumePlainPutFastPath = false) {
+            if (("operation" in prepared &&
+                !(prepared.operation instanceof PutOperation)) ||
+                (!assumePlainPutFastPath &&
+                    !this.canUsePlainPutFastPath(prepared.document, options))) {
+                return;
+            }
+            const indexedContextNext = existingHead
+                ? this.nextFromIndexedContext(existingHead, existingLocalContext)
+                : undefined;
+            const next = existingHead
+                ? indexedContextNext
+                    ? [indexedContextNext]
+                    : [await this._resolveEntry(existingHead)]
+                : [];
+            const canCleanupTrimmedHeads = this._index.canGetIndexedKeyByHead();
+            return {
+                document: prepared.document,
+                encodedDocument: prepared.encodedDocument,
+                payloadData: "operationPayloadBytes" in prepared
+                    ? prepared.operationPayloadBytes
+                    : (prepared.encodedOperation ??
+                        encodePutOperationPayload(prepared.operation.data)),
+                key: prepared.key,
+                operation: "operation" in prepared ? prepared.operation : undefined,
+                next,
+                skipMissingNextJoin: !options?.checkRemote,
+                resolveTrimmedEntries: !canCleanupTrimmedHeads,
+                useGenericChangeHandler: !options?.unique && existingLocalContext === undefined,
+                unique: options?.unique,
+                existing: existingLocalContext,
+            };
+        }
+        commitPlainPutPlan(plan, options) {
+            return mapMaybePromise(this.commitNativeDocumentAppend({
+                document: plan.document,
+                key: plan.key,
+                operation: plan.operation,
+                documentBytes: plan.encodedDocument,
+                operationPayloadBytes: plan.payloadData,
+                next: plan.next,
+                skipMissingNextJoin: plan.skipMissingNextJoin,
+                resolveTrimmedEntries: plan.resolveTrimmedEntries,
+                options,
+                unique: plan.unique,
+                existing: plan.existing,
+            }), (documentAppendCommit) => {
+                const handled = plan.useGenericChangeHandler
+                    ? this.handleChanges({
+                        added: [{ head: true, entry: documentAppendCommit.entry }],
+                        removed: documentAppendCommit.removed,
+                    }, {
+                        document: plan.document,
+                        operation: documentAppendCommit.operation ??
+                            plan.operation ??
+                            new PutOperation({ data: plan.encodedDocument }),
+                        key: plan.key,
+                        unique: plan.unique,
+                        existing: plan.existing,
+                    })
+                    : this.handlePreparedPlainPutCommit(documentAppendCommit);
+                return mapMaybePromise(handled, () => {
+                    this.keepCache?.add(documentAppendCommit.append.hash);
+                    return {
+                        get entry() {
+                            return documentAppendCommit.entry;
+                        },
+                        removed: documentAppendCommit.removed,
+                    };
+                });
+            });
+        }
+        commitNativeDocumentAppend(input) {
+            const trustedLog = asTrustedDocumentSharedLog(this.log);
+            const appendOptions = {
+                ...input.options,
+                meta: {
+                    next: input.next,
+                    ...input.options?.meta,
+                },
+                replicate: input.options?.replicate,
+            };
+            const prepareNativeDocumentIndexWithAppendFacts = this.createNativeBackboneDocumentIndexAppendFactsPreparer(input);
+            const preferAppendFactsDocumentIndex = this.isNativeMode() && !!prepareNativeDocumentIndexWithAppendFacts;
+            return mapMaybePromise(preferAppendFactsDocumentIndex
+                ? undefined
+                : this.prepareNativeBackboneDocumentIndexCommit(input), (nativeDocumentIndexCommit) => {
+                let committedNativeDocumentIndex = nativeDocumentIndexCommit;
+                const prepareNativeDocumentIndexWithAppendFactsForCommit = nativeDocumentIndexCommit
+                    ? undefined
+                    : prepareNativeDocumentIndexWithAppendFacts;
+                if (this.isNativeMode() &&
+                    !nativeDocumentIndexCommit &&
+                    !prepareNativeDocumentIndexWithAppendFactsForCommit) {
+                    throw this.nativeModeError("requires native document-index commit");
+                }
+                const appendProperties = {
+                    skipMissingNextJoin: input.skipMissingNextJoin,
+                    resolveTrimmedEntries: input.resolveTrimmedEntries,
+                    payloadData: input.operationPayloadBytes,
+                    useNativeExistingDocumentContext: input.useNativeExistingDocumentContext,
+                    ...(nativeDocumentIndexCommit
+                        ? {
+                            nativeBackboneDocumentIndex: this.toNativeBackboneDocumentIndexCommitInput(input, nativeDocumentIndexCommit),
+                        }
+                        : {}),
+                    ...(prepareNativeDocumentIndexWithAppendFactsForCommit
+                        ? {
+                            prepareNativeBackboneDocumentIndex: (facts) => {
+                                committedNativeDocumentIndex =
+                                    prepareNativeDocumentIndexWithAppendFactsForCommit(facts);
+                                return committedNativeDocumentIndex
+                                    ? this.toNativeBackboneDocumentIndexCommitInput(input, committedNativeDocumentIndex)
+                                    : undefined;
+                            },
+                        }
+                        : {}),
+                };
+                if (input.operation) {
+                    if (this.isNativeMode()) {
+                        throw this.nativeModeError("requires payload-backed put operations");
+                    }
+                    return mapMaybePromise(trustedLog.appendLocallyPrepared(input.operation, appendOptions, appendProperties), (appended) => this.createNativeCheckedDocumentAppendCommitFacts(input, appended, committedNativeDocumentIndex));
+                }
+                const commitOnlyAppend = this.isNativeMode()
+                    ? trustedLog.appendStrictNativeDocumentPayloadCommitOnly(input.operationPayloadBytes, appendOptions, appendProperties)
+                    : trustedLog.appendLocallyPreparedPayloadCommitOnly(input.operationPayloadBytes, appendOptions, appendProperties);
+                return mapMaybePromise(commitOnlyAppend, (commitOnly) => {
+                    if (commitOnly) {
+                        return this.createNativeCheckedDocumentAppendCommitFacts(input, commitOnly, committedNativeDocumentIndex);
+                    }
+                    if (this.isNativeMode()) {
+                        throw this.nativeModeError("requires native payload commit-only append");
+                    }
+                    return this.commitNativeDocumentAppendPayloadFallback(input, appendOptions, appendProperties, committedNativeDocumentIndex);
+                });
+            });
+        }
+        createNativeCheckedDocumentAppendCommitFacts(input, appended, nativeBackboneDocumentIndex) {
+            return mapMaybePromise(this.createDocumentAppendCommitFacts(input, appended, nativeBackboneDocumentIndex), (commit) => {
+                this.assertNativeModeDocumentAppendCommit(commit);
+                return commit;
+            });
+        }
+        assertNativeModeDocumentAppendCommit(commit) {
+            if (!this.isNativeMode()) {
+                return;
+            }
+            if (!commit.nativeBackboneDocumentIndexCommitted) {
+                throw this.nativeModeError("requires native document-index commit");
+            }
+        }
+        toNativeBackboneDocumentIndexCommitInput(input, commit, useLatestContext = false) {
+            const canUsePlainPutPayload = commit.usePlainPutPayload === true ||
+                (!!input.operationPayloadBytes && !!commit.projection);
+            return {
+                key: documentIndexStoreKey(input.key),
+                valuePrefixBytes: commit.valuePrefixBytes,
+                usePlainPutPayload: canUsePlainPutPayload,
+                projection: commit.projection,
+                existingCreated: input.unique || input.existing === null
+                    ? undefined
+                    : input.existing?.value.__context.created,
+                deleteTrimmedHeads: !this.hasDocumentChangeConsumers() &&
+                    this._index.canGetIndexedKeyByHead(),
+                useLatestContext,
+                requiredPreviousSignerPublicKey: input.requiredPreviousSignerPublicKey,
+            };
+        }
+        prepareNativeBackboneDocumentIndexCommit(input) {
+            if (!this._nativeBackboneDocumentIndexEnabled) {
+                return;
+            }
+            return asTrustedDocumentIndex(this._index).prepareNativeBackboneDocumentIndexCommit(input.document, input.documentBytes, { entryPublicKeys: [this.log.log.identity.publicKey] });
+        }
+        createNativeBackboneDocumentIndexAppendFactsPreparer(input) {
+            if (!this._nativeBackboneDocumentIndexEnabled ||
+                !asTrustedDocumentIndex(this._index).canPrepareNativeBackboneDocumentIndexCommitWithAppendFacts()) {
+                return;
+            }
+            const existing = input.unique || input.existing === null ? null : input.existing;
+            return (facts) => {
+                const appendFacts = {
+                    wallTime: BigInt(facts.wallTime),
+                    gid: facts.gid,
+                    payloadSize: facts.payloadSize,
+                };
+                const context = nativeDocumentContextFactsAsContext({
+                    created: existing?.value.__context.created || appendFacts.wallTime,
+                    modified: appendFacts.wallTime,
+                    head: "",
+                    gid: appendFacts.gid,
+                    size: appendFacts.payloadSize,
+                });
+                return asTrustedDocumentIndex(this._index).prepareNativeBackboneDocumentIndexCommitWithAppendFacts(input.document, input.documentBytes, context, { entryPublicKeys: [this.log.log.identity.publicKey] });
+            };
+        }
+        async commitNativeDocumentAppendPayloadFallback(input, appendOptions, appendProperties, nativeBackboneDocumentIndex) {
+            if (this.isNativeMode()) {
+                throw this.nativeModeError("requires native payload append support");
+            }
+            const trustedLog = asTrustedDocumentSharedLog(this.log);
+            let appended;
+            try {
+                appended = await trustedLog.appendLocallyPreparedPayload(input.operationPayloadBytes, appendOptions, appendProperties);
+            }
+            catch (error) {
+                if (!(error instanceof Error) ||
+                    error.message !==
+                        "appendLocallyPrepared payload-only path requires native append support") {
+                    throw error;
+                }
+                appended = await trustedLog.appendLocallyPrepared(new PutOperation({ data: input.documentBytes }), appendOptions, appendProperties);
+            }
+            return this.createDocumentAppendCommitFacts(input, appended, nativeBackboneDocumentIndex);
+        }
+        async commitNativeDocumentAppendMany(input) {
+            const trustedLog = asTrustedDocumentSharedLog(this.log);
+            const nativeBackboneDocumentIndexes = await this.prepareNativeBackboneDocumentIndexCommitBatch(input.puts);
+            const nativeBackboneDocumentIndexInputs = nativeBackboneDocumentIndexes?.map((commit, index) => this.toNativeBackboneDocumentIndexCommitInput(input.puts[index], commit, input.useNativeExistingDocumentContext === true));
+            const nexts = input.puts.map((put) => {
+                if (input.useNativeExistingDocumentContext === true) {
+                    return [];
+                }
+                const existing = put.unique || put.existing === null ? null : put.existing;
+                if (!existing) {
+                    return [];
+                }
+                const context = existing.value.__context;
+                const next = this.nextFromIndexedContext(context.head, existing);
+                if (!next) {
+                    throw this.nativeModeError("requires indexed document context for non-unique putMany");
+                }
+                return [next];
+            });
+            const appended = await trustedLog.appendLocallyPreparedPayloadsManyIndependent(input.puts.map((put) => put.operationPayloadBytes), {
+                ...input.options,
+                replicate: input.options?.replicate,
+            }, {
+                resolveTrimmedEntries: input.resolveTrimmedEntries,
+                nexts,
+                nativeBackboneDocumentIndexes: nativeBackboneDocumentIndexInputs,
+                retainMaterializationBytes: this._hasLogTrim,
+            });
+            if (!appended) {
+                if (this.isNativeMode()) {
+                    throw this.nativeModeError("requires native batched payload append support");
+                }
+                return undefined;
+            }
+            const appendInputs = input.puts.map((put, index) => ({
+                input: nativeBackboneDocumentIndexes?.[index]
+                    ? {
+                        ...put,
+                        nativeBackboneDocumentIndex: nativeBackboneDocumentIndexes[index],
+                    }
+                    : put,
+                appended: (() => {
+                    const materializeEntry = appended.materializeEntries?.[index];
+                    let entry;
+                    return {
+                        get entry() {
+                            return (entry ??= materializeEntry
+                                ? materializeEntry()
+                                : appended.entries[index]);
+                        },
+                        removed: [],
+                        appendCommit: appended.appendCommits[index],
+                    };
+                })(),
+            }));
+            const commits = await this.createDocumentAppendCommitFactsBatch(appendInputs);
+            let entries;
+            return {
+                get entries() {
+                    return (entries ??= commits.map((commit) => commit.entry));
+                },
+                removed: appended.removed,
+                commits,
+            };
+        }
+        prepareNativeBackboneDocumentIndexCommitBatch(inputs) {
+            if (!this._nativeBackboneDocumentIndexEnabled || inputs.length === 0) {
+                return;
+            }
+            const commits = [];
+            const finishAsync = (firstAsyncIndex, firstAsyncCommit) => Promise.all([
+                firstAsyncCommit,
+                ...inputs
+                    .slice(firstAsyncIndex + 1)
+                    .map((input) => this.prepareNativeBackboneDocumentIndexCommit(input)),
+            ]).then((resolvedCommits) => {
+                for (const commit of resolvedCommits) {
+                    if (!commit) {
+                        return;
+                    }
+                    commits.push(commit);
+                }
+                return commits;
+            });
+            for (let i = 0; i < inputs.length; i++) {
+                const commit = this.prepareNativeBackboneDocumentIndexCommit(inputs[i]);
+                if (isPromiseLike(commit)) {
+                    return finishAsync(i, commit);
+                }
+                if (!commit) {
+                    return;
+                }
+                commits.push(commit);
+            }
+            return commits;
+        }
+        createDocumentAppendCommitFacts(input, appended, nativeBackboneDocumentIndex) {
+            const append = appended.appendCommit;
+            const nativePreviousContext = append.documentPreviousContext == null
+                ? undefined
+                : nativeDocumentContextFactsAsContext(append.documentPreviousContext);
+            const nativePreviousIndexedContext = nativePreviousContext
+                ? {
+                    id: input.key,
+                    value: {
+                        __context: nativePreviousContext,
+                    },
+                }
+                : undefined;
+            const inputWithExisting = input.existing === undefined && nativePreviousIndexedContext
+                ? {
+                    ...input,
+                    existing: nativePreviousIndexedContext,
+                }
+                : input;
+            const existing = inputWithExisting.unique || inputWithExisting.existing === null
+                ? null
+                : inputWithExisting.existing;
+            const contextInput = {
+                existingCreated: existing?.value.__context.created,
+                modified: append.wallTime,
+                head: append.hash,
+                gid: append.gid,
+                size: append.payloadSize,
+            };
+            if (append.nativeBackboneDocumentIndexCommitted) {
+                return this.createDocumentAppendCommitFactsWithLazyContext(inputWithExisting, appended, contextInput, nativeBackboneDocumentIndex);
+            }
+            const contextPlan = tryPlanDocumentContext(contextInput);
+            if (contextPlan) {
+                return this.createDocumentAppendCommitFactsWithContext(inputWithExisting, appended, contextPlan, nativeBackboneDocumentIndex);
+            }
+            return planDocumentContext(contextInput).then((plannedContext) => plannedContext
+                ? this.createDocumentAppendCommitFactsWithContext(inputWithExisting, appended, plannedContext, nativeBackboneDocumentIndex)
+                : this.createDocumentAppendCommitFactsWithLazyContext(inputWithExisting, appended, contextInput, nativeBackboneDocumentIndex));
+        }
+        createDocumentAppendCommitFactsWithLazyContext(input, appended, contextInput, nativeBackboneDocumentIndex = input.nativeBackboneDocumentIndex) {
+            let contextValues;
+            let context;
+            let contextBytes;
+            const getContextValues = () => {
+                if (contextValues) {
+                    return contextValues;
+                }
+                const modified = toContextBigInt(contextInput.modified);
+                const existingCreated = contextInput.existingCreated == null
+                    ? undefined
+                    : toContextBigInt(contextInput.existingCreated);
+                return (contextValues = {
+                    created: existingCreated == null || existingCreated === 0n
+                        ? modified
+                        : existingCreated,
+                    modified,
+                    head: contextInput.head,
+                    gid: contextInput.gid,
+                    size: contextInput.size,
+                });
+            };
+            const getContext = () => (context ??= new Context(getContextValues()));
+            const getContextBytes = () => (contextBytes ??= encodeDocumentContextSuffix(getContext()));
+            return this.createNativeDocumentAppendTransaction(input, appended, {
+                getContext,
+                getContextBytes,
+            }, nativeBackboneDocumentIndex);
+        }
+        createNativeDocumentAppendTransaction(input, appended, contextAccessors, nativeBackboneDocumentIndex = input.nativeBackboneDocumentIndex) {
+            const append = appended.appendCommit;
+            let contextualEncodedValueParts;
+            let exposedNativeBackboneDocumentIndex;
+            let nativeBackboneDocumentIndexContextSet = false;
+            const ensureNativeBackboneDocumentIndexContext = () => {
+                if (nativeBackboneDocumentIndexContextSet ||
+                    !nativeBackboneDocumentIndex?.setContext) {
+                    return;
+                }
+                nativeBackboneDocumentIndex.setContext(contextAccessors.getContext());
+                nativeBackboneDocumentIndexContextSet = true;
+            };
+            const getNativeBackboneDocumentIndex = () => {
+                if (!nativeBackboneDocumentIndex) {
+                    return;
+                }
+                return (exposedNativeBackboneDocumentIndex ??= {
+                    valuePrefixBytes: nativeBackboneDocumentIndex.valuePrefixBytes,
+                    projection: nativeBackboneDocumentIndex.projection,
+                    indexable: nativeBackboneDocumentIndex.indexable,
+                    getIndexable: nativeBackboneDocumentIndex.getIndexable
+                        ? () => {
+                            ensureNativeBackboneDocumentIndexContext();
+                            return nativeBackboneDocumentIndex.getIndexable();
+                        }
+                        : undefined,
+                    setContext: nativeBackboneDocumentIndex.setContext
+                        ? (context) => {
+                            nativeBackboneDocumentIndex.setContext(context);
+                            nativeBackboneDocumentIndexContextSet = true;
+                        }
+                        : undefined,
+                });
+            };
+            return {
+                document: input.document,
+                key: input.key,
+                operation: input.operation,
+                encodedDocument: input.documentBytes,
+                operationPayloadBytes: input.operationPayloadBytes,
+                get entry() {
+                    return appended.entry;
+                },
+                removed: appended.removed,
+                removedHashes: appended.removedHashes,
+                append,
+                coordinateFields: append.coordinateFields,
+                get context() {
+                    return contextAccessors.getContext();
+                },
+                get contextBytes() {
+                    return contextAccessors.getContextBytes();
+                },
+                get contextualEncodedValueParts() {
+                    return (contextualEncodedValueParts ??= {
+                        prefix: input.documentBytes,
+                        suffix: contextAccessors.getContextBytes(),
+                    });
+                },
+                nativeBackboneDocumentIndexCommitted: appended.appendCommit.nativeBackboneDocumentIndexCommitted,
+                nativeBackboneDocumentIndexTrimmedHeadsProcessed: appended.appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed,
+                get nativeBackboneDocumentIndex() {
+                    return getNativeBackboneDocumentIndex();
+                },
+                unique: input.unique,
+                existing: input.existing,
+            };
+        }
+        async createDocumentAppendCommitFactsBatch(rows) {
+            const contextInputs = rows.map(({ input, appended }) => {
+                const append = appended.appendCommit;
+                const nativePreviousContext = append.documentPreviousContext == null
+                    ? undefined
+                    : nativeDocumentContextFactsAsContext(append.documentPreviousContext);
+                const nativePreviousIndexedContext = nativePreviousContext
+                    ? {
+                        id: input.key,
+                        value: {
+                            __context: nativePreviousContext,
+                        },
+                    }
+                    : undefined;
+                const inputWithExisting = input.existing === undefined && nativePreviousIndexedContext
+                    ? {
+                        ...input,
+                        existing: nativePreviousIndexedContext,
+                    }
+                    : input;
+                const existing = inputWithExisting.unique || inputWithExisting.existing === null
+                    ? null
+                    : inputWithExisting.existing;
+                return {
+                    existingCreated: existing?.value.__context.created,
+                    modified: append.wallTime,
+                    head: append.hash,
+                    gid: append.gid,
+                    size: append.payloadSize,
+                };
+            });
+            const contextPlans = tryPlanDocumentContextBatch(contextInputs) ??
+                (await planDocumentContextBatch(contextInputs));
+            return rows.map((row, index) => {
+                const append = row.appended.appendCommit;
+                const nativePreviousContext = append.documentPreviousContext == null
+                    ? undefined
+                    : nativeDocumentContextFactsAsContext(append.documentPreviousContext);
+                const nativePreviousIndexedContext = nativePreviousContext
+                    ? {
+                        id: row.input.key,
+                        value: {
+                            __context: nativePreviousContext,
+                        },
+                    }
+                    : undefined;
+                const input = row.input.existing === undefined && nativePreviousIndexedContext
+                    ? {
+                        ...row.input,
+                        existing: nativePreviousIndexedContext,
+                    }
+                    : row.input;
+                const contextPlan = contextPlans?.[index];
+                if (!contextPlan) {
+                    return this.createDocumentAppendCommitFactsWithLazyContext(input, row.appended, contextInputs[index], input.nativeBackboneDocumentIndex);
+                }
+                if (input.nativeBackboneDocumentIndex) {
+                    let context;
+                    return this.createNativeDocumentAppendTransaction(input, row.appended, {
+                        getContext: () => (context ??= new Context(contextPlan)),
+                        getContextBytes: () => contextPlan.contextBytes,
+                    }, input.nativeBackboneDocumentIndex);
+                }
+                return this.createDocumentAppendCommitFactsWithContext(input, row.appended, contextPlan);
+            });
+        }
+        createDocumentAppendCommitFactsWithContext(input, appended, contextPlan, preparedNativeBackboneDocumentIndex = input.nativeBackboneDocumentIndex) {
+            const append = appended.appendCommit;
+            const context = new Context(contextPlan);
+            const nativeBackboneDocumentIndex = preparedNativeBackboneDocumentIndex ??
+                (append.nativeBackboneDocumentIndexCommitted
+                    ? undefined
+                    : this._nativeBackboneDocumentIndexEnabled
+                        ? asTrustedDocumentIndex(this._index).prepareNativeBackboneDocumentIndexCommitWithAppendFacts(input.document, input.documentBytes, context, { entryPublicKeys: [this.log.log.identity.publicKey] })
+                        : undefined);
+            return this.createNativeDocumentAppendTransaction(input, appended, {
+                getContext: () => context,
+                getContextBytes: () => contextPlan.contextBytes,
+            }, nativeBackboneDocumentIndex);
+        }
+        hasDocumentChangeConsumers() {
+            if (this._valueClassIsProgram === true) {
+                // Program-valued documents must always materialize removed values so
+                // that open subprograms are dropped on delete, even without listeners.
+                return true;
+            }
+            const changeListenerCount = this._documentChangeListenerCount ?? 0;
+            const internalChangeListenerCount = this._documentInternalChangeListenerCount ?? 0;
+            return (changeListenerCount > internalChangeListenerCount ||
+                this._index.hasPending === true);
+        }
+        dispatchDocumentChangeIfObserved(documentsChanged) {
+            if (!this.hasDocumentChangeConsumers()) {
+                return;
+            }
+            this.events.dispatchEvent(new CustomEvent("change", { detail: documentsChanged }));
+        }
+        nextFromIndexedContext(existingHead, existing) {
+            const context = existing?.value.__context;
+            if (!context || context.head !== existingHead) {
+                return;
+            }
+            return new ShallowEntry({
+                hash: context.head,
+                head: false,
+                payloadSize: context.size,
+                meta: new ShallowMeta({
+                    gid: context.gid,
+                    clock: new LamportClock({
+                        id: this.log.log.identity.publicKey.bytes,
+                        timestamp: new Timestamp({
+                            wallTime: context.modified,
+                            logical: 0,
+                        }),
+                    }),
+                    next: [],
+                    type: EntryType.APPEND,
+                }),
+            });
+        }
+        handlePreparedPlainPutCommit(commit) {
+            const shouldPrepareChange = this.hasDocumentChangeConsumers();
+            const removedAlreadyHandled = commit.nativeBackboneDocumentIndexTrimmedHeadsProcessed === true;
+            const removedHashes = commit.removedHashes ?? [];
+            const hasRemovedFacts = commit.removed.length > 0 || removedHashes.length > 0;
+            const existing = commit.unique || commit.existing === null ? null : commit.existing;
+            const persistNativeBackboneDocumentIndexCommit = () => {
+                if (!commit.nativeBackboneDocumentIndexCommitted) {
+                    return;
+                }
+                if (this._mode === "native") {
+                    return true;
+                }
+                return asTrustedDocumentIndex(this._index)._persistPreparedNativeBackboneDocumentIndexStoredWithContext(commit.key, commit.context, commit.nativeBackboneDocumentIndex, commit.contextualEncodedValueParts, {
+                    replace: existing != null,
+                });
+            };
+            if (!shouldPrepareChange &&
+                (!hasRemovedFacts || removedAlreadyHandled) &&
+                commit.nativeBackboneDocumentIndexCommitted) {
+                if (!this.strictHistory && existing) {
+                    const shouldIgnoreChange = this.immutable
+                        ? existing.value.__context.modified < commit.append.wallTime
+                        : existing.value.__context.modified > commit.append.wallTime;
+                    if (shouldIgnoreChange) {
+                        return;
+                    }
+                }
+                const finishCommitted = () => {
+                    this._index._cacheResolvedIdentityValue(commit.key.primitive, commit.document);
+                };
+                const persisted = persistNativeBackboneDocumentIndexCommit();
+                return persisted === undefined || persisted === false
+                    ? finishCommitted()
+                    : mapMaybePromise(persisted, finishCommitted);
+            }
+            const documentsChanged = shouldPrepareChange
+                ? {
+                    added: [],
+                    removed: [],
+                }
+                : undefined;
+            const modified = new Set();
+            if (!this.strictHistory && existing) {
+                const shouldIgnoreChange = this.immutable
+                    ? existing.value.__context.modified < commit.append.wallTime
+                    : existing.value.__context.modified > commit.append.wallTime;
+                if (shouldIgnoreChange) {
+                    modified.add(commit.key.primitive);
+                }
+            }
+            const finishRemoved = () => {
+                if (!shouldPrepareChange) {
+                    if (!hasRemovedFacts || removedAlreadyHandled) {
+                        return undefined;
+                    }
+                    if (commit.removed.length === 0 && removedHashes.length > 0) {
+                        const handled = this.tryHandlePreparedPlainPutCommitRemovedHashesFromHeads(removedHashes, modified);
+                        if (handled !== undefined) {
+                            return mapMaybePromise(handled, (handledHeads) => {
+                                if (handledHeads.size === removedHashes.length) {
+                                    return undefined;
+                                }
+                                const remaining = removedHashes.filter((hash) => !handledHeads.has(hash));
+                                return remaining.length === 0
+                                    ? undefined
+                                    : this.handlePreparedPlainPutCommitRemovedHashes(remaining, modified);
+                            });
+                        }
+                        return this.handlePreparedPlainPutCommitRemovedHashes(removedHashes, modified);
+                    }
+                    const handled = this.tryHandlePreparedPlainPutCommitRemovedFromHeads(commit.removed, modified);
+                    if (handled !== undefined) {
+                        return mapMaybePromise(handled, (handledHeads) => {
+                            if (handledHeads.size === commit.removed.length) {
+                                return undefined;
+                            }
+                            const remaining = commit.removed.filter((entry) => !handledHeads.has(entry.hash));
+                            return remaining.length === 0
+                                ? undefined
+                                : this.handlePreparedPlainPutCommitRemoved(remaining, modified);
+                        });
+                    }
+                    return this.handlePreparedPlainPutCommitRemoved(commit.removed, modified);
+                }
+                if (commit.removed.length === 0) {
+                    if (removedHashes.length > 0) {
+                        return this.handlePreparedPlainPutCommitRemovedHashes(removedHashes, modified, documentsChanged);
+                    }
+                    this.dispatchDocumentChangeIfObserved(documentsChanged);
+                    return;
+                }
+                return this.handlePreparedPlainPutCommitRemoved(commit.removed, modified, documentsChanged);
+            };
+            const finishIndexed = (indexedDocument) => {
+                if (indexedDocument) {
+                    if (documentsChanged) {
+                        documentsChanged.added.push(indexedDocument);
+                    }
+                    modified.add(commit.key.primitive);
+                    return finishRemoved();
+                }
+                return mapMaybePromise(this._index.putWithContext(commit.document, commit.key, commit.context, {
+                    replace: existing != null,
+                    encodedValueParts: commit.contextualEncodedValueParts,
+                    transformFacts: { entryPublicKeys: commit.entry.publicKeys },
+                }), ({ indexable }) => {
+                    if (documentsChanged) {
+                        documentsChanged.added.push(coerceWithIndexed(coerceWithContext(commit.document, commit.context), indexable));
+                    }
+                    modified.add(commit.key.primitive);
+                    return finishRemoved();
+                });
+            };
+            if (!modified.has(commit.key.primitive)) {
+                if (commit.nativeBackboneDocumentIndexCommitted) {
+                    const finishCommitted = () => {
+                        this._index._cacheResolvedIdentityValue(commit.key.primitive, commit.document);
+                        if (!shouldPrepareChange) {
+                            modified.add(commit.key.primitive);
+                            return finishRemoved();
+                        }
+                        const withContext = coerceWithContext(commit.document, commit.context);
+                        if (commit.nativeBackboneDocumentIndex?.indexable) {
+                            return finishIndexed(coerceWithIndexed(withContext, commit.nativeBackboneDocumentIndex.indexable));
+                        }
+                        if (commit.nativeBackboneDocumentIndex?.getIndexable) {
+                            return finishIndexed(coerceWithLazyIndexed(withContext, commit.nativeBackboneDocumentIndex.getIndexable));
+                        }
+                        return finishIndexed(coerceWithIndexed(withContext, commit.document));
+                    };
+                    const persisted = persistNativeBackboneDocumentIndexCommit();
+                    return persisted === undefined || persisted === false
+                        ? finishCommitted()
+                        : mapMaybePromise(persisted, finishCommitted);
+                }
+                if (commit.nativeBackboneDocumentIndex) {
+                    const nativePreparedIndexPut = asTrustedDocumentIndex(this._index)._putPreparedNativeBackboneDocumentIndexWithContext(commit.document, commit.key, commit.context, commit.nativeBackboneDocumentIndex, {
+                        replace: existing != null,
+                    });
+                    if (nativePreparedIndexPut !== undefined) {
+                        return mapMaybePromise(nativePreparedIndexPut, finishIndexed);
+                    }
+                }
+                const storedIdentityPut = this._index._putStoredIdentityWithContext(commit.document, commit.key, commit.context, commit.contextualEncodedValueParts, {
+                    replace: existing != null,
+                });
+                if (storedIdentityPut !== undefined) {
+                    return mapMaybePromise(storedIdentityPut, finishIndexed);
+                }
+                return mapMaybePromise(this._index._putIdentityWithContext(commit.document, commit.key, commit.context, {
+                    replace: existing != null,
+                    encodedValueParts: commit.contextualEncodedValueParts,
+                    transformFacts: { entryPublicKeys: commit.entry.publicKeys },
+                }), finishIndexed);
+            }
+            return finishRemoved();
+        }
+        tryHandlePreparedPlainPutCommitRemovedFromHeads(removedEntries, modified) {
+            const handled = new Set();
+            const deleteKeys = [];
+            for (const removed of removedEntries) {
+                if (removed instanceof Entry) {
+                    continue;
+                }
+                const resolved = this._index.tryGetIdentityIndexedKeyByHead(removed.hash);
+                if (!resolved.supported) {
+                    return;
+                }
+                if (!resolved.key) {
+                    continue;
+                }
+                handled.add(removed.hash);
+                if (modified.has(resolved.key.primitive)) {
+                    continue;
+                }
+                deleteKeys.push(resolved.key);
+                modified.add(resolved.key.primitive);
+            }
+            if (deleteKeys.length === 0) {
+                return handled;
+            }
+            return mapMaybePromise(this._index.delManyMaybe(deleteKeys), () => handled);
+        }
+        tryHandlePreparedPlainPutCommitRemovedHashesFromHeads(removedHashes, modified) {
+            const handled = new Set();
+            const deleteKeys = [];
+            for (const hash of removedHashes) {
+                const resolved = this._index.tryGetIdentityIndexedKeyByHead(hash);
+                if (!resolved.supported) {
+                    return;
+                }
+                if (!resolved.key) {
+                    continue;
+                }
+                handled.add(hash);
+                if (modified.has(resolved.key.primitive)) {
+                    continue;
+                }
+                deleteKeys.push(resolved.key);
+                modified.add(resolved.key.primitive);
+            }
+            if (deleteKeys.length === 0) {
+                return handled;
+            }
+            return mapMaybePromise(this._index.delManyMaybe(deleteKeys), () => handled);
+        }
+        async handlePreparedPlainPutCommitRemoved(removedEntries, modified, documentsChanged) {
+            const handledRemovedHeads = await this.collectRemovedDocumentChangesFromIndexedHeads(removedEntries, modified, documentsChanged);
+            for (const removed of removedEntries) {
+                if (handledRemovedHeads.has(removed.hash)) {
+                    continue;
+                }
+                if (!(removed instanceof Entry) &&
+                    (await this.collectRemovedDocumentChangeFromIndexedHead(removed.hash, modified, documentsChanged))) {
+                    continue;
+                }
+                const entry = removed instanceof Entry
+                    ? removed
+                    : await this.log.log.entryIndex.get(removed.hash, {
+                        type: "full",
+                        ignoreMissing: true,
+                    });
+                if (!entry) {
+                    continue;
+                }
+                try {
+                    const payload = await this.getAppendOperation(entry);
+                    if (!payload) {
+                        continue;
+                    }
+                    await this.collectRemovedDocumentChange(payload, modified, documentsChanged);
+                }
+                catch (error) {
+                    if (error instanceof AccessError) {
+                        continue;
+                    }
+                    throw error;
+                }
+            }
+            if (documentsChanged) {
+                this.dispatchDocumentChangeIfObserved(documentsChanged);
+            }
+        }
+        async handlePreparedPlainPutCommitRemovedHashes(removedHashes, modified, documentsChanged) {
+            const handledRemovedHeads = await this.collectRemovedDocumentChangesFromIndexedHeadHashes(removedHashes, modified, documentsChanged);
+            for (const hash of removedHashes) {
+                if (handledRemovedHeads.has(hash)) {
+                    continue;
+                }
+                const entry = await this.log.log.entryIndex.get(hash, {
+                    type: "full",
+                    ignoreMissing: true,
+                });
+                if (!entry) {
+                    continue;
+                }
+                try {
+                    const payload = await this.getAppendOperation(entry);
+                    if (!payload) {
+                        continue;
+                    }
+                    await this.collectRemovedDocumentChange(payload, modified, documentsChanged);
+                }
+                catch (error) {
+                    if (error instanceof AccessError) {
+                        continue;
+                    }
+                    throw error;
+                }
+            }
+            if (documentsChanged) {
+                this.dispatchDocumentChangeIfObserved(documentsChanged);
+            }
+        }
+        async handlePreparedPlainPutManyCommit(commit) {
+            if (!this.hasDocumentChangeConsumers() &&
+                commit.removed.length === 0) {
+                const stored = await asTrustedDocumentIndex(this._index)._putManyPreparedNativeBackboneDocumentIndexStored(commit.commits.map((put) => {
+                    const existing = put.unique || put.existing === null ? null : put.existing;
+                    return {
+                        value: put.document,
+                        id: put.key,
+                        context: put.context,
+                        encodedValueParts: put.contextualEncodedValueParts,
+                        nativeDocumentIndex: put.nativeBackboneDocumentIndex,
+                        options: {
+                            replace: existing != null,
+                        },
+                    };
+                }));
+                if (stored === true) {
+                    return;
+                }
+            }
+            const documentsChanged = {
+                added: [],
+                removed: [],
+            };
+            const modified = new Set();
+            const putsToIndex = [];
+            for (const put of commit.commits) {
+                if (modified.has(put.key.primitive)) {
+                    continue;
+                }
+                const existing = put.unique || put.existing === null ? null : put.existing;
+                putsToIndex.push({
+                    document: put.document,
+                    encodedDocument: put.encodedDocument,
+                    key: put.key,
+                    context: put.context,
+                    contextualEncodedValueParts: put.contextualEncodedValueParts,
+                    nativeBackboneDocumentIndex: put.nativeBackboneDocumentIndex,
+                    replace: existing != null,
+                });
+                modified.add(put.key.primitive);
+            }
+            let indexedDocuments = await this._index._putManyIdentityWithContext(putsToIndex.map((put) => ({
+                value: put.document,
+                id: put.key,
+                context: put.context,
+                options: {
+                    replace: put.replace,
+                    encodedValueParts: put.contextualEncodedValueParts,
+                },
+            })));
+            indexedDocuments ??=
+                await asTrustedDocumentIndex(this._index)._putManyPreparedNativeBackboneDocumentIndexWithContext(putsToIndex.map((put) => ({
+                    value: put.document,
+                    id: put.key,
+                    context: put.context,
+                    nativeDocumentIndex: put.nativeBackboneDocumentIndex,
+                    options: {
+                        replace: put.replace,
+                    },
+                })));
+            if (indexedDocuments) {
+                documentsChanged.added.push(...indexedDocuments);
+            }
+            else {
+                if (this.isNativeMode()) {
+                    throw this.nativeModeError("requires native batch document-index commit");
+                }
+                const indexed = await this._index.putManyWithContext(putsToIndex.map((put) => ({
+                    value: put.document,
+                    id: put.key,
+                    context: put.context,
+                    options: {
+                        replace: put.replace,
+                        encodedValueParts: put.contextualEncodedValueParts,
+                    },
+                })));
+                for (let i = 0; i < putsToIndex.length; i++) {
+                    const put = putsToIndex[i];
+                    const { indexable } = indexed[i];
+                    documentsChanged.added.push(coerceWithIndexed(coerceWithContext(put.document, put.context), indexable));
+                }
+            }
+            const handledRemovedHeads = await this.collectRemovedDocumentChangesFromIndexedHeads(commit.removed, modified, documentsChanged);
+            for (const removed of commit.removed) {
+                if (handledRemovedHeads.has(removed.hash)) {
+                    continue;
+                }
+                if (!(removed instanceof Entry) &&
+                    (await this.collectRemovedDocumentChangeFromIndexedHead(removed.hash, modified, documentsChanged))) {
+                    continue;
+                }
+                const entry = removed instanceof Entry
+                    ? removed
+                    : await this.log.log.entryIndex.get(removed.hash, {
+                        type: "full",
+                        ignoreMissing: true,
+                    });
+                if (!entry) {
+                    continue;
+                }
+                try {
+                    const payload = await this.getAppendOperation(entry);
+                    if (!payload) {
+                        continue;
+                    }
+                    await this.collectRemovedDocumentChange(payload, modified, documentsChanged);
+                }
+                catch (error) {
+                    if (error instanceof AccessError) {
+                        continue;
+                    }
+                    throw error;
                 }
-                throw error;
             }
-            return true;
+            this.events.dispatchEvent(new CustomEvent("change", { detail: documentsChanged }));
         }
-        async _canAppend(entry, reference) {
-            const resolve = async (history) => {
-                return typeof history === "string"
-                    ? this.log.log.get(history) ||
-                        (await Entry.fromMultihash(this.log.log.blocks, history))
-                    : history;
-            };
-            const pointsToHistory = async (history) => {
-                // make sure nexts only points to this document at some point in history
-                let current = await resolve(history);
-                const next = entry.meta.next[0];
-                while (current?.hash &&
-                    next !== current?.hash &&
-                    current.meta.next.length > 0) {
-                    current = await this.log.log.get(current.meta.next[0]);
-                }
-                if (current?.hash === next) {
-                    return true; // Ok, we are pointing this new edit to some exising point in time of the old document
+        async collectRemovedDocumentChangeFromIndexedHead(head, modified, documentsChanged) {
+            const key = await this._index.getIdentityIndexedKeyByHead(head);
+            if (key) {
+                if (await this.collectRemovedDocumentChangeFromIndexedKey(key, modified, documentsChanged)) {
+                    return true;
                 }
+            }
+            if (!documentsChanged) {
                 return false;
-            };
-            try {
-                entry.init({
-                    encoding: this.log.log.encoding,
-                    keychain: this.node.services.keychain,
-                });
-                const operation = reference?.operation ||
-                    /* entry._payload instanceof DecryptedThing
-                        ? entry.payload.getValue(entry.encoding)
-                        :  */ (await entry.getPayloadValue()); // TODO implement sync api for resolving entries that does not deep decryption
-                if (isPutOperation(operation)) {
-                    // check nexts
-                    const putOperation = operation;
-                    let value = reference?.document ??
-                        this.index.valueEncoding.decoder(putOperation.data);
-                    const keyValue = this.idResolver(value);
-                    const key = indexerTypes.toId(keyValue);
-                    const existingDocument = this.immutable
-                        ? (await this.index.getDetailed(key, {
-                            resolve: false,
+            }
+            const indexed = await this._index.getIdentityIndexedByHead(head);
+            if (!indexed) {
+                return false;
+            }
+            return this.collectRemovedDocumentChangeFromIndexedKey(indexed.id, modified, documentsChanged, () => coerceWithIndexed(indexed.value, indexed.value));
+        }
+        async collectRemovedDocumentChangeFromIndexedKey(key, modified, documentsChanged, valueProvider) {
+            if (modified.has(key.primitive)) {
+                return true;
+            }
+            let value;
+            if (documentsChanged) {
+                value =
+                    valueProvider?.() ??
+                        (await this._index.get(key, {
                             local: true,
-                            remote: { strategy: "fallback" },
-                        }))?.[0]?.results[0]
-                        : await this.getLocalIndexedContext(key);
-                    const existingContext = this.getExistingContext(existingDocument);
-                    if (existingContext && existingContext.head !== entry.hash) {
-                        //  econd condition can false if we reset the operation log, while not  resetting the index. For example when doing .recover
-                        if (this.immutable) {
-                            // key already exist but pick the oldest entry
-                            // this is because we can not overwrite same id if immutable
-                            if (existingContext.created < entry.meta.clock.timestamp.wallTime) {
-                                return false;
-                            }
-                            if (entry.meta.next.length > 0) {
-                                return false; // can not append to immutable document
-                            }
-                            return putOperation;
-                        }
-                        else {
-                            if (this.strictHistory) {
-                                // make sure that the next pointer exist and points to the existing documents
-                                if (entry.meta.next.length !== 1) {
-                                    return false;
-                                }
-                                const prevEntry = await this.log.log.entryIndex.get(existingContext.head);
-                                if (!prevEntry) {
-                                    logger.error("Failed to find previous entry for document edit: " +
-                                        entry.hash);
-                                    return false;
-                                }
-                                const referenceHistoryCorrectly = await pointsToHistory(prevEntry);
-                                return referenceHistoryCorrectly ? putOperation : false;
-                            }
-                            else {
-                                return putOperation;
-                            }
-                        }
+                            remote: false,
+                        }));
+                if (!value) {
+                    return false;
+                }
+                documentsChanged.removed.push(value);
+            }
+            if (value instanceof Program &&
+                value.closed !== true &&
+                value.parents.includes(this)) {
+                await value.drop(this);
+            }
+            await this._index.delMany([key]);
+            modified.add(key.primitive);
+            return true;
+        }
+        async collectRemovedDocumentChangesFromIndexedHeads(removed, modified, documentsChanged) {
+            return this.collectRemovedDocumentChangesFromIndexedHeadHashes(removed.map((entry) => entry.hash), modified, documentsChanged);
+        }
+        async collectRemovedDocumentChangesFromIndexedHeadHashes(removedHashes, modified, documentsChanged) {
+            if (removedHashes.length === 0) {
+                return new Set();
+            }
+            if (!documentsChanged) {
+                const handled = new Set();
+                const deleteKeys = [];
+                for (const hash of removedHashes) {
+                    const key = await this._index.getIdentityIndexedKeyByHead(hash);
+                    if (!key) {
+                        continue;
                     }
-                    else {
-                        // TODO should re reject next pointers to other documents?
-                        // like if (entry.meta.next.length > 0) { return false; }
-                        // for now the default behaviour will allow us to build document dependencies
+                    handled.add(hash);
+                    if (modified.has(key.primitive)) {
+                        continue;
                     }
+                    deleteKeys.push(key);
+                    modified.add(key.primitive);
                 }
-                else if (isDeleteOperation(operation)) {
-                    if (entry.meta.next.length !== 1) {
-                        return false;
-                    }
-                    const existingDocument = this.immutable
-                        ? (await this.index.getDetailed(operation.key, {
-                            resolve: false,
-                            local: true,
-                            remote: true,
-                        }))?.[0]?.results[0]
-                        : await this.getLocalIndexedContext(operation.key instanceof indexerTypes.IdKey
-                            ? operation.key
-                            : indexerTypes.toId(operation.key));
-                    const existingHead = this.getExistingContext(existingDocument)?.head;
-                    if (!existingHead) {
-                        // already deleted
-                        return coerceDeleteOperation(operation); // assume ok
+                await this._index.delMany(deleteKeys);
+                return handled;
+            }
+            const keyByHead = this._index.getIndexedKeysByHeads(removedHashes);
+            if (keyByHead) {
+                const handled = new Set();
+                const deleteKeys = [];
+                for (let i = 0; i < removedHashes.length; i++) {
+                    const key = keyByHead[i];
+                    if (!key) {
+                        continue;
                     }
-                    let doc = await this.log.log.get(existingHead);
-                    if (!doc) {
-                        logger.error("Failed to find Document from head");
-                        return false;
+                    handled.add(removedHashes[i]);
+                    if (modified.has(key.primitive)) {
+                        continue;
                     }
-                    if (await pointsToHistory(doc)) {
-                        // references the existing document
-                        return coerceDeleteOperation(operation);
+                    const value = await this._index.get(key, {
+                        local: true,
+                        remote: false,
+                    });
+                    if (!value) {
+                        handled.delete(removedHashes[i]);
+                        continue;
                     }
-                    return false;
+                    documentsChanged.removed.push(value);
+                    deleteKeys.push(key);
+                    modified.add(key.primitive);
                 }
-                else {
-                    throw new Error("Unsupported operation");
-                }
-                return operation;
+                await this._index.delMany(deleteKeys);
+                return handled;
             }
-            catch (error) {
-                if (error instanceof AccessError) {
-                    return false; // we cant index because we can not decrypt
+            const indexedByHead = await this._index.getIdentityIndexedByHeads(removedHashes);
+            if (!indexedByHead) {
+                return new Set();
+            }
+            const handled = new Set();
+            const deleteKeys = [];
+            for (let i = 0; i < removedHashes.length; i++) {
+                const indexed = indexedByHead[i];
+                if (!indexed) {
+                    continue;
                 }
-                else if (error instanceof BorshError) {
-                    warn("Received payload that could not be decoded, skipping");
-                    return false;
+                const key = indexed.id;
+                handled.add(removedHashes[i]);
+                if (modified.has(key.primitive)) {
+                    continue;
                 }
-                throw error;
+                if (documentsChanged) {
+                    const value = coerceWithIndexed(indexed.value, indexed.value);
+                    documentsChanged.removed.push(value);
+                }
+                deleteKeys.push(key);
+                modified.add(key.primitive);
             }
+            await this._index.delMany(deleteKeys);
+            return handled;
         }
-        async put(doc, options) {
-            const keyValue = this.idResolver(doc);
-            // type check the key
-            indexerTypes.checkId(keyValue);
-            const ser = serialize(doc);
-            if (ser.length > MAX_BATCH_SIZE) {
-                throw new Error(`Document is too large (${ser.length * 1e-6}) mb). Needs to be less than ${MAX_BATCH_SIZE * 1e-6} mb`);
-            }
-            const existingHead = options?.unique
-                ? undefined
-                : options?.checkRemote
-                    ? (await this._index.getDetailed(keyValue, {
-                        resolve: false,
+        async collectRemovedDocumentChange(payload, modified, documentsChanged) {
+            let value;
+            let key;
+            if (isPutOperation(payload)) {
+                const keyValue = this.isNativeMode()
+                    ? await this.getNativeDocumentIdFromPutOperation(payload)
+                    : undefined;
+                if (this.isNativeMode() && keyValue == null) {
+                    throw this.nativeModeError("requires native document id extraction for removed put");
+                }
+                key = indexerTypes.toId(keyValue ??
+                    this.idResolver(this.index.valueEncoding.decoder(payload.data)));
+                if (modified.has(key.primitive)) {
+                    return;
+                }
+                if (documentsChanged) {
+                    const document = await this._index.get(key, {
                         local: true,
-                        remote: { replicate: options?.replicate },
-                    }))?.[0]?.results[0]?.context.head
-                    : (await this.getLocalIndexedContext(indexerTypes.toId(keyValue)))
-                        ?.value.__context.head;
-            let operation;
-            if (this.compatibility === 6) {
-                if (typeof keyValue === "string") {
-                    operation = new PutWithKeyOperation({
-                        key: keyValue,
-                        data: ser,
+                        remote: false,
                     });
+                    if (!document) {
+                        return;
+                    }
+                    value = document;
                 }
-                else {
-                    throw new Error("Key must be a string in compatibility mode v6");
+            }
+            else if (isDeleteOperation(payload)) {
+                key = coerceDeleteOperation(payload).key;
+                if (modified.has(key.primitive)) {
+                    return;
+                }
+                if (documentsChanged) {
+                    const document = await this._index.get(key, {
+                        local: true,
+                        remote: false,
+                    });
+                    if (!document) {
+                        return;
+                    }
+                    value = document;
                 }
             }
             else {
-                operation = new PutOperation({
-                    data: ser,
-                });
+                throw new Error("Unexpected");
             }
-            const appended = await this.log.append(operation, {
-                ...options,
-                meta: {
-                    next: existingHead ? [await this._resolveEntry(existingHead)] : [],
-                    ...options?.meta,
-                },
-                canAppend: (entry) => {
-                    return this.canAppend(entry, { document: doc, operation });
-                },
-                onChange: (change) => {
-                    return this.handleChanges(change, { document: doc, operation });
-                },
-                replicate: options?.replicate,
+            if (documentsChanged && value) {
+                documentsChanged.removed.push(value);
+            }
+            if (value instanceof Program &&
+                value.closed !== true &&
+                value.parents.includes(this)) {
+                await value.drop(this);
+            }
+            await this._index.delMany([key]);
+            modified.add(key.primitive);
+        }
+        async collectRemovedPutChangeFromNativeId(payload, modified) {
+            if (!this.isNativeMode() || !isPutOperation(payload)) {
+                return false;
+            }
+            const keyValue = await this.getNativeDocumentIdFromPutOperation(payload);
+            if (keyValue == null) {
+                throw this.nativeModeError("requires native document id extraction for removed put");
+            }
+            const key = indexerTypes.toId(keyValue);
+            if (modified.has(key.primitive)) {
+                return true;
+            }
+            await this._index.delMany([key]);
+            modified.add(key.primitive);
+            return true;
+        }
+        putStrictNativeReceivedDocumentIndexWithContext(value, key, entry, payload, existing) {
+            if (!this.isNativeMode() || !this._nativeBackboneDocumentIndexEnabled) {
+                return;
+            }
+            if (value instanceof Program) {
+                return;
+            }
+            const existingContext = this.getExistingContext(existing);
+            const modified = entry.meta.clock.timestamp.wallTime;
+            const context = new Context({
+                created: existingContext?.created || modified,
+                modified,
+                head: entry.hash,
+                gid: entry.meta.gid,
+                size: encodePutOperationPayload(payload.data).byteLength,
+            });
+            const nativeDocumentIndex = asTrustedDocumentIndex(this._index).prepareNativeBackboneDocumentIndexCommitWithAppendFacts(value, payload.data, context, { entryPublicKeys: entry.publicKeys });
+            if (!nativeDocumentIndex) {
+                return;
+            }
+            return asTrustedDocumentIndex(this._index)._putPreparedNativeBackboneDocumentIndexWithContext(value, key, context, nativeDocumentIndex, {
+                replace: existing != null,
+            });
+        }
+        putStrictNativeReceivedDocumentIndexStoredWithContext(key, entry, payload, existing) {
+            if (!this.isNativeMode() || !this._nativeBackboneDocumentIndexEnabled) {
+                return;
+            }
+            const existingContext = this.getExistingContext(existing);
+            const modified = entry.meta.clock.timestamp.wallTime;
+            const context = new Context({
+                created: existingContext?.created || modified,
+                modified,
+                head: entry.hash,
+                gid: entry.meta.gid,
+                size: encodePutOperationPayload(payload.data).byteLength,
+            });
+            const nativeDocumentIndex = asTrustedDocumentIndex(this._index).prepareNativeBackboneDocumentIndexStoredCommitWithAppendFacts(payload.data, context, { entryPublicKeys: entry.publicKeys });
+            if (!nativeDocumentIndex) {
+                return;
+            }
+            return asTrustedDocumentIndex(this._index)._putPreparedNativeBackboneDocumentIndexStoredWithContext(key, context, nativeDocumentIndex, {
+                replace: existing != null,
             });
-            this.keepCache?.add(appended.entry.hash);
-            return appended;
         }
         async get(id, options) {
             const resolved = await this.index.get(id, {
@@ -480,6 +3070,9 @@ let Documents = (() => {
             return resolved ? resolved : undefined;
         }
         async del(id, options) {
+            return this._documentBackend.del(id, options);
+        }
+        async delCompatDocumentBackend(id, options) {
             const key = id instanceof indexerTypes.IdKey ? id : indexerTypes.toId(id);
             const existing = (await this._index.getDetailed(key, {
                 resolve: false,
@@ -504,6 +3097,100 @@ let Documents = (() => {
                 },
             });
         }
+        async delNativeDocumentBackend(id, options) {
+            const deleteOptions = this.normalizeNativeModePutOptions(options);
+            this.assertNativeModeDeleteSupported(deleteOptions);
+            const key = id instanceof indexerTypes.IdKey ? id : indexerTypes.toId(id);
+            if (!this.hasNativeDocumentContextLookup()) {
+                throw this.nativeModeError("requires native document context lookup");
+            }
+            const existing = this.getNativeIndexedContext(key);
+            const existingContext = this.getExistingContext(existing);
+            if (!existingContext?.head) {
+                throw new NotFoundError(`No entry with key '${key.primitive}' in the database`);
+            }
+            let previousEntry;
+            let existingDocumentChecked = false;
+            let existingDocument;
+            const getPreviousEntry = async () => {
+                if (previousEntry) {
+                    return previousEntry;
+                }
+                previousEntry = await this._resolveEntry(existingContext.head, {
+                    remote: true,
+                });
+                if (!previousEntry) {
+                    throw new NotFoundError(`No entry with key '${key.primitive}' in the database`);
+                }
+                return previousEntry;
+            };
+            const getExistingDocument = async () => {
+                if (!existingDocumentChecked) {
+                    existingDocumentChecked = true;
+                    existingDocument = await this.getLocalIdentityDocumentByHead(existingContext.head);
+                    existingDocument ??=
+                        await this.getLocalIndexedDocumentForNativeDeletePolicy(key);
+                }
+                return existingDocument;
+            };
+            const operation = new DeleteOperation({ key });
+            if (!(await this.canPerformAllowsNativeDelete({
+                operation,
+                getExistingEntry: getPreviousEntry,
+                getExistingDocument,
+            }))) {
+                throw this.nativeModeError("canPerform policy rejected this delete");
+            }
+            const operationPayloadBytes = BORSH_ENCODING_OPERATION.encoder(operation);
+            const documentsChanged = this.hasDocumentChangeConsumers()
+                ? {
+                    added: [],
+                    removed: [],
+                }
+                : undefined;
+            const removedDocument = documentsChanged
+                ? await this._index.get(key, {
+                    local: true,
+                    remote: false,
+                })
+                : undefined;
+            this.keepCache?.delete(existingContext.head);
+            const previousForAppend = this.nextFromIndexedContext(existingContext.head, existing) ??
+                (await getPreviousEntry());
+            const trustedLog = asTrustedDocumentSharedLog(this.log);
+            const appended = await trustedLog.appendStrictNativeDocumentPayloadCommitOnly(operationPayloadBytes, {
+                ...deleteOptions,
+                meta: {
+                    next: [previousForAppend],
+                    type: EntryType.CUT,
+                    ...deleteOptions?.meta,
+                },
+            }, {
+                skipMissingNextJoin: true,
+                resolveTrimmedEntries: false,
+                nativeBackboneDocumentDeleteKey: documentIndexStoreKey(key),
+            });
+            if (!appended) {
+                throw this.nativeModeError("requires native delete append support");
+            }
+            const result = {
+                get entry() {
+                    return appended.entry;
+                },
+                removed: appended.removed,
+            };
+            if (appended.appendCommit.nativeBackboneDocumentDeleteCommitted) {
+                this._index.clearResolvedCacheForKeys([key]);
+            }
+            else {
+                await this._index.delManyMaybe([key]);
+            }
+            if (documentsChanged && removedDocument) {
+                documentsChanged.removed.push(removedDocument);
+                this.dispatchDocumentChangeIfObserved(documentsChanged);
+            }
+            return result;
+        }
         async handleChanges(change, reference) {
             logger.trace("handleChanges called", change);
             const isAppendOperation = change?.added.length === 1 ? !!change.added[0] : false;
@@ -511,45 +3198,90 @@ let Documents = (() => {
             for (const r of change.removed) {
                 removedSet.set(r.hash, r);
             }
+            const shouldPrepareDocumentChanges = this.hasDocumentChangeConsumers();
+            const canRemoveByIndexedHead = !shouldPrepareDocumentChanges && this._index.canGetIndexedKeyByHead();
+            const removedEntries = canRemoveByIndexedHead
+                ? []
+                : ((await Promise.all(change.removed.map((x) => x instanceof Entry ? x : this.log.log.entryIndex.get(x.hash)))) ?? []);
             const sortedEntries = [
                 ...change.added.map((x) => x.entry),
-                ...((await Promise.all(change.removed.map((x) => x instanceof Entry ? x : this.log.log.entryIndex.get(x.hash)))) || []),
+                ...removedEntries,
             ]; // TODO assert sorting
             /*  const sortedEntries = [...change.added, ...(removed || [])]
                         .sort(this.log.log.sortFn)
                         .reverse(); // sort so we get newest to oldest */
             // There might be a case where change.added and change.removed contains the same document id. Usaully because you use the "trim" option
             // in combinatpion with inserting the same document. To mitigate this, we loop through the changes and modify the behaviour for this
-            let documentsChanged = {
-                added: [],
-                removed: [],
-            };
+            let documentsChanged = shouldPrepareDocumentChanges
+                ? {
+                    added: [],
+                    removed: [],
+                }
+                : undefined;
             let modified = new Set();
             for (const item of sortedEntries) {
                 if (!item) {
                     continue;
                 }
                 try {
-                    const payload = 
-                    /* item._payload instanceof DecryptedThing
-                        ? item.payload.getValue(item.encoding)
-                        :  */ await item.getPayloadValue(); // TODO implement sync api for resolving entries that does not deep decryption
+                    const isReferencedAppendEntry = isAppendOperation &&
+                        reference?.operation &&
+                        change.added[0]?.entry.hash === item.hash;
+                    const payload = isReferencedAppendEntry
+                        ? reference.operation
+                        : await this.getAppendOperation(item);
+                    if (!payload) {
+                        continue;
+                    }
                     if (isPutOperation(payload) && !removedSet.has(item.hash)) {
-                        let value = (isAppendOperation &&
-                            reference?.operation === payload &&
-                            reference?.document) ||
+                        if (!documentsChanged && this.isNativeMode()) {
+                            const keyValue = await this.getNativeDocumentIdFromPutOperation(payload);
+                            if (keyValue != null) {
+                                const key = indexerTypes.toId(keyValue);
+                                if (modified.has(key.primitive)) {
+                                    continue;
+                                }
+                                const existing = reference?.unique || reference?.existing === null
+                                    ? null
+                                    : isReferencedAppendEntry &&
+                                        reference?.existing !== undefined
+                                        ? reference.existing
+                                        : this.getNativeModeIndexedContext(key) || null;
+                                if (!this.strictHistory && existing) {
+                                    const shouldIgnoreChange = this.immutable
+                                        ? existing.value.__context.modified <
+                                            item.meta.clock.timestamp.wallTime
+                                        : existing.value.__context.modified >
+                                            item.meta.clock.timestamp.wallTime;
+                                    if (shouldIgnoreChange) {
+                                        continue;
+                                    }
+                                }
+                                const stored = await this.putStrictNativeReceivedDocumentIndexStoredWithContext(key, item, payload, existing);
+                                if (stored) {
+                                    modified.add(key.primitive);
+                                    continue;
+                                }
+                            }
+                        }
+                        let value = (isReferencedAppendEntry && reference?.document) ||
                             this.index.valueEncoding.decoder(payload.data);
                         // get index key from value
-                        const keyObject = this.idResolver(value);
-                        const key = indexerTypes.toId(keyObject);
+                        const key = isReferencedAppendEntry && reference?.key
+                            ? reference.key
+                            : indexerTypes.toId(this.idResolver(value));
                         // document is already updated with more recent entry
                         if (modified.has(key.primitive)) {
                             continue;
                         }
                         // if no casual ordering is used, use timestamps to order docs
-                        let existing = reference?.unique
+                        let existing = reference?.unique || reference?.existing === null
                             ? null
-                            : (await this.getLocalIndexedContext(key)) || null;
+                            : isReferencedAppendEntry && reference?.existing !== undefined
+                                ? reference.existing
+                                : this.isNativeMode()
+                                    ? this.getNativeModeIndexedContext(key) || null
+                                    : (await this.getLocalIndexedContext(key)) || null;
                         if (!this.strictHistory && existing) {
                             // if immutable use oldest, else use newest
                             let shouldIgnoreChange = this.immutable
@@ -566,59 +3298,26 @@ let Documents = (() => {
                             // if replicator, then open
                             value = await this.maybeSubprogramOpen(value);
                         }
+                        const nativeStoredIndexed = payload instanceof PutOperation
+                            ? await this.putStrictNativeReceivedDocumentIndexWithContext(value, key, item, payload, existing)
+                            : undefined;
+                        if (nativeStoredIndexed) {
+                            documentsChanged?.added.push(nativeStoredIndexed);
+                            modified.add(key.primitive);
+                            continue;
+                        }
                         const { context, indexable } = await this._index.put(value, key, item, existing);
-                        documentsChanged.added.push(coerceWithIndexed(coerceWithContext(value, context), indexable));
+                        documentsChanged?.added.push(coerceWithIndexed(coerceWithContext(value, context), indexable));
                         modified.add(key.primitive);
                     }
                     else if ((isDeleteOperation(payload) && !removedSet.has(item.hash)) ||
                         isPutOperation(payload) ||
                         removedSet.has(item.hash)) {
-                        let value;
-                        let key;
-                        if (isPutOperation(payload)) {
-                            const valueWithoutContext = this.index.valueEncoding.decoder(payload.data);
-                            key = indexerTypes.toId(this.idResolver(valueWithoutContext));
-                            // document is already updated with more recent entry
-                            if (modified.has(key.primitive)) {
-                                continue;
-                            }
-                            // we try to fetch it anyway, because we need the context for the events
-                            const document = await this._index.get(key, {
-                                local: true,
-                                remote: false,
-                            });
-                            if (!document) {
-                                continue;
-                            }
-                            value = document;
-                        }
-                        else if (isDeleteOperation(payload)) {
-                            key = coerceDeleteOperation(payload).key;
-                            // document is already updated with more recent entry
-                            if (modified.has(key.primitive)) {
-                                continue;
-                            }
-                            const document = await this._index.get(key, {
-                                local: true,
-                                remote: false,
-                            });
-                            if (!document) {
-                                continue;
-                            }
-                            value = document;
-                        }
-                        else {
-                            throw new Error("Unexpected");
-                        }
-                        documentsChanged.removed.push(value);
-                        if (value instanceof Program &&
-                            value.closed !== true &&
-                            value.parents.includes(this)) {
-                            await value.drop(this);
+                        if (!documentsChanged &&
+                            (await this.collectRemovedPutChangeFromNativeId(payload, modified))) {
+                            continue;
                         }
-                        // update index
-                        await this._index.del(key);
-                        modified.add(key.primitive);
+                        await this.collectRemovedDocumentChange(payload, modified, documentsChanged);
                     }
                     else {
                         // Unknown operation
@@ -632,7 +3331,39 @@ let Documents = (() => {
                     throw error;
                 }
             }
-            this.events.dispatchEvent(new CustomEvent("change", { detail: documentsChanged }));
+            if (canRemoveByIndexedHead && change.removed.length > 0) {
+                const handled = await this.collectRemovedDocumentChangesFromIndexedHeads(change.removed, modified);
+                const remainingRemoved = change.removed.filter((entry) => !handled.has(entry.hash));
+                for (const removed of remainingRemoved) {
+                    const entry = removed instanceof Entry
+                        ? removed
+                        : await this.log.log.entryIndex.get(removed.hash, {
+                            type: "full",
+                            ignoreMissing: true,
+                        });
+                    if (!entry) {
+                        continue;
+                    }
+                    try {
+                        const payload = await this.getAppendOperation(entry);
+                        if (!payload) {
+                            continue;
+                        }
+                        if (!(await this.collectRemovedPutChangeFromNativeId(payload, modified))) {
+                            await this.collectRemovedDocumentChange(payload, modified);
+                        }
+                    }
+                    catch (error) {
+                        if (error instanceof AccessError) {
+                            continue;
+                        }
+                        throw error;
+                    }
+                }
+            }
+            if (documentsChanged) {
+                this.dispatchDocumentChangeIfObserved(documentsChanged);
+            }
         }
         async count(options) {
             // Local/exact count