@peerbit/log 6.1.0 → 6.2.0

package/dist/src/entry-index.js

@@ -3,14 +3,49 @@ import {} from "@peerbit/blocks-interface";
 import { Cache } from "@peerbit/cache";
 import { BoolQuery, Not, Or, Sort, SortDirection, StringMatch, StringMatchMethod, toId, } from "@peerbit/indexer-interface";
 import { FOREGROUND_READ_MESSAGE_PRIORITY } from "@peerbit/stream-interface";
+import { LamportClock as Clock, Timestamp } from "./clock.js";
+import { ShallowEntry, ShallowMeta } from "./entry-shallow.js";
 import { EntryType } from "./entry-type.js";
-import { Entry } from "./entry.js";
+import { Entry, } from "./entry.js";
 import { logger as baseLogger } from "./logger.js";
 const log = baseLogger.newScope("entry-index");
 const LOG_ENTRY_REMOTE_READ_PRIORITY = FOREGROUND_READ_MESSAGE_PRIORITY;
 const ENTRY_CACHE_MAX_SIZE = 10; // TODO as param for log
 const DEFERRED_INDEX_FLUSH_IDLE_MS = 250;
 const NATIVE_GRAPH_REBUILD_BATCH_SIZE = 512;
+const TIMESTAMP_WALL_TIME_KEY = ["meta", "clock", "timestamp", "wallTime"];
+const TIMESTAMP_LOGICAL_KEY = ["meta", "clock", "timestamp", "logical"];
+const entryIndexProfileNow = () => globalThis.performance?.now?.() ?? Date.now();
+const entryIndexProfileStart = (sink) => sink ? entryIndexProfileNow() : 0;
+const emitEntryIndexProfileDuration = (sink, startedAt, event) => {
+    if (!sink) {
+        return;
+    }
+    sink({
+        ...event,
+        durationMs: entryIndexProfileNow() - startedAt,
+    });
+};
+const hasRmMany = (store) => typeof store.rmMany === "function";
+const hasExactDelete = (index) => typeof index.delIds === "function";
+const hasExactDeleteCount = (index) => typeof index.delIdsCount === "function";
+const putPreparedEntryBlock = async (store, prepared) => {
+    const storeWithKnown = store;
+    return typeof storeWithKnown.putKnown === "function"
+        ? await storeWithKnown.putKnown(prepared.cid, prepared.block.bytes)
+        : await store.put(prepared);
+};
+const isPromiseLike = (value) => !!value && typeof value.then === "function";
+const mapMaybePromise = (value, fn) => (isPromiseLike(value) ? value.then(fn) : fn(value));
+const materializePreparedAppendShallowEntry = (entry, isHead) => {
+    const shallowEntry = entry.shallowEntry ?? entry.getShallowEntry?.(isHead);
+    if (!shallowEntry) {
+        throw new Error("Missing prepared append shallow entry");
+    }
+    shallowEntry.head = isHead;
+    entry.shallowEntry = shallowEntry;
+    return shallowEntry;
+};
 const withDefaultRemoteReadPriority = (options) => {
     if (!options?.remote) {
         return options;
@@ -32,6 +67,27 @@ const withDefaultRemoteReadPriority = (options) => {
         },
     };
 };
+const canBatchResolveFromStore = (store, options) => {
+    if (typeof store.getMany !== "function") {
+        return false;
+    }
+    return typeof options !== "object" || !options.remote;
+};
+const sortKeyEquals = (sort, key) => Array.isArray(sort?.key) &&
+    sort.key.length === key.length &&
+    sort.key.every((part, index) => part === key[index]);
+const timestampSortDirection = (sort) => {
+    if (sort.length < 2) {
+        return;
+    }
+    const [wallTime, logical] = sort;
+    if (!sortKeyEquals(wallTime, TIMESTAMP_WALL_TIME_KEY) ||
+        !sortKeyEquals(logical, TIMESTAMP_LOGICAL_KEY) ||
+        wallTime.direction !== logical.direction) {
+        return;
+    }
+    return wallTime.direction;
+};
 export class EntryIndex {
     properties;
     cache;
@@ -41,6 +97,7 @@ export class EntryIndex {
     insertionPromises;
     pendingIndexWrites;
     pendingIndexFlushTimer;
+    pendingIndexFlushLastWriteMs = 0;
     constructor(properties) {
         this.properties = properties;
         this.sortReversed = properties.sort.sort.map((x) => deserialize(serialize(x), Sort));
@@ -53,15 +110,39 @@ export class EntryIndex {
         this.insertionPromises = new Map();
         this.pendingIndexWrites = new Map();
     }
+    materializePendingIndexWrite(hash, pending) {
+        if (typeof pending !== "function") {
+            return pending;
+        }
+        const shallowEntry = pending();
+        this.pendingIndexWrites.set(hash, shallowEntry);
+        return shallowEntry;
+    }
+    getPendingIndexWrite(hash) {
+        const pending = this.pendingIndexWrites.get(hash);
+        return pending
+            ? this.materializePendingIndexWrite(hash, pending)
+            : undefined;
+    }
     schedulePendingIndexWriteFlush() {
+        this.pendingIndexFlushLastWriteMs = Date.now();
         if (this.pendingIndexFlushTimer) {
-            clearTimeout(this.pendingIndexFlushTimer);
+            return;
         }
-        this.pendingIndexFlushTimer = setTimeout(() => {
+        const flushAfterIdle = () => {
+            const remainingMs = DEFERRED_INDEX_FLUSH_IDLE_MS -
+                (Date.now() - this.pendingIndexFlushLastWriteMs);
+            if (remainingMs > 0) {
+                this.pendingIndexFlushTimer = setTimeout(flushAfterIdle, remainingMs);
+                this.pendingIndexFlushTimer.unref?.();
+                return;
+            }
+            this.pendingIndexFlushTimer = undefined;
             void this.flushPendingWrites().catch((error) => {
                 log.error("Failed to flush deferred entry-index writes", error);
             });
-        }, DEFERRED_INDEX_FLUSH_IDLE_MS);
+        };
+        this.pendingIndexFlushTimer = setTimeout(flushAfterIdle, DEFERRED_INDEX_FLUSH_IDLE_MS);
         this.pendingIndexFlushTimer.unref?.();
     }
     clearPendingIndexFlushTimer() {
@@ -79,13 +160,27 @@ export class EntryIndex {
             return;
         }
         this.clearPendingIndexFlushTimer();
+        const writes = [];
         for (const hash of keys) {
             const pending = this.pendingIndexWrites.get(hash);
             if (!pending) {
                 continue;
             }
-            await this.properties.index.put(pending);
-            this.pendingIndexWrites.delete(hash);
+            writes.push(this.materializePendingIndexWrite(hash, pending));
+        }
+        if (writes.length === 0) {
+            return;
+        }
+        if (writes.length > 1 && this.properties.index.putBatch) {
+            await this.properties.index.putBatch(writes);
+        }
+        else {
+            for (const write of writes) {
+                await this.properties.index.put(write);
+            }
+        }
+        for (const write of writes) {
+            this.pendingIndexWrites.delete(write.hash);
         }
         if (this.pendingIndexWrites.size > 0) {
             this.schedulePendingIndexWriteFlush();
@@ -93,6 +188,15 @@ export class EntryIndex {
     }
     getHeads(gid, resolve = false) {
         if (this.properties.nativeGraph?.useHeads) {
+            const shape = getResolveShape(resolve);
+            if (shape && isHeadHashOnlyShape(shape)) {
+                return this.iterateNativeProjected(() => this.properties
+                    .nativeGraph.graph.heads(gid)
+                    .map((hash) => ({ hash })), shape);
+            }
+            if (shape && isHeadDataShape(shape)) {
+                return this.iterateNativeProjected(() => this.properties.nativeGraph.graph.headDataEntries(gid), shape);
+            }
             return this.iterateNativeHashes(() => this.properties.nativeGraph.graph.heads(gid), resolve);
         }
         const query = [];
@@ -107,6 +211,110 @@ export class EntryIndex {
         }
         return this.iterate(query, undefined, resolve);
     }
+    getHeadsForAppend(gid) {
+        if (!this.properties.nativeGraph?.useHeads) {
+            return undefined;
+        }
+        return this.properties.nativeGraph.graph.headEntries(gid);
+    }
+    async hasHead(gid) {
+        if (!this.properties.nativeGraph?.useHeads) {
+            return undefined;
+        }
+        return this.properties.nativeGraph.graph.hasHead(gid);
+    }
+    async hasAnyHead(gids) {
+        if (!this.properties.nativeGraph?.useHeads) {
+            return undefined;
+        }
+        const uniqueGids = new Set([...gids].filter(Boolean));
+        if (uniqueGids.size === 0) {
+            return false;
+        }
+        return this.properties.nativeGraph.graph.hasAnyHead(uniqueGids);
+    }
+    async hasAnyHeadBatch(gidSets) {
+        if (!this.properties.nativeGraph?.useHeads) {
+            return undefined;
+        }
+        const normalized = [...gidSets].map((gids) => new Set([...gids].filter(Boolean)));
+        if (normalized.length === 0) {
+            return [];
+        }
+        return this.properties.nativeGraph.graph.hasAnyHeadBatch(normalized);
+    }
+    async getMaxHeadDataU32(gid) {
+        if (!this.properties.nativeGraph?.useHeads) {
+            return undefined;
+        }
+        return this.properties.nativeGraph.graph.maxHeadDataU32(gid);
+    }
+    async getMaxHeadDataU32Batch(gids) {
+        if (!this.properties.nativeGraph?.useHeads) {
+            return undefined;
+        }
+        const normalized = [...gids];
+        if (normalized.length === 0) {
+            return [];
+        }
+        return this.properties.nativeGraph.graph.maxHeadDataU32Batch?.(normalized);
+    }
+    getJoinHeads(gid) {
+        if (this.properties.nativeGraph?.useHeads) {
+            return Promise.resolve(this.properties.nativeGraph.graph.joinHeadEntries(gid));
+        }
+        return this.getHeads(gid, {
+            type: "shape",
+            shape: {
+                hash: true,
+                meta: { type: true, next: true, gid: true, clock: true },
+            },
+        }).all();
+    }
+    getJoinChildren(next) {
+        if (this.properties.nativeGraph) {
+            return Promise.resolve(this.properties.nativeGraph.graph.childJoinEntries(next));
+        }
+        return this.getHasNext(next, {
+            type: "shape",
+            shape: {
+                hash: true,
+                meta: { type: true, next: true, gid: true, clock: true },
+            },
+        }).all();
+    }
+    getUniqueReferenceGids(hash) {
+        if (!this.properties.nativeGraph) {
+            return undefined;
+        }
+        return this.properties.nativeGraph.graph.uniqueReferenceGids(hash);
+    }
+    getUniqueReferenceGidRowsBatch(hashes) {
+        if (!this.properties.nativeGraph) {
+            return undefined;
+        }
+        const normalized = [...hashes];
+        if (normalized.length === 0) {
+            return [];
+        }
+        return this.properties.nativeGraph.graph.uniqueReferenceGidRowsBatch?.(normalized);
+    }
+    getUniqueReferenceGidRowsFlatBatch(hashes) {
+        if (!this.properties.nativeGraph) {
+            return undefined;
+        }
+        const normalized = [...hashes];
+        if (normalized.length === 0) {
+            return [];
+        }
+        return this.properties.nativeGraph.graph.uniqueReferenceGidRowsFlatBatch?.(normalized);
+    }
+    planDeleteRecursively(from, skipFirst = false) {
+        if (!this.properties.nativeGraph) {
+            return undefined;
+        }
+        return this.properties.nativeGraph.graph.planDeleteRecursively([...from].map((entry) => entry.hash), skipFirst);
+    }
     getHasNext(next, resolve) {
         const query = [
             new StringMatch({
@@ -122,10 +330,10 @@ export class EntryIndex {
         return this._countHasNext(next, excludeHash);
     }
     async _countHasNext(next, excludeHash = undefined) {
-        await this.flushPendingWrites();
         if (this.properties.nativeGraph) {
             return this.properties.nativeGraph.graph.countHasNext(next, excludeHash);
         }
+        await this.flushPendingWrites();
         const query = [
             new StringMatch({
                 key: ["meta", "next"],
@@ -161,13 +369,13 @@ export class EntryIndex {
         let complete = false;
         const getHashes = async () => {
             if (!hashPromise) {
-                hashPromise = this.flushPendingWrites().then(() => hashes());
+                hashPromise = Promise.resolve(hashes());
             }
             return hashPromise;
         };
         const coerce = async (hashes) => {
             if (resolveInFull) {
-                const resolved = await Promise.all(hashes.map((hash) => this.resolve(hash, resolveInFullOptions)));
+                const resolved = await this.resolveMany(hashes, resolveInFullOptions);
                 return resolved.filter((entry) => !!entry);
             }
             const shallow = await Promise.all(hashes.map((hash) => this.getShallow(hash)));
@@ -194,6 +402,36 @@ export class EntryIndex {
             },
         };
     }
+    iterateNativeProjected(values, shape) {
+        let valuesPromise;
+        let offset = 0;
+        let complete = false;
+        const getValues = async () => {
+            if (!valuesPromise) {
+                valuesPromise = Promise.resolve(values());
+            }
+            return valuesPromise;
+        };
+        const coerce = (values) => values.map((value) => projectShape(value, shape));
+        return {
+            close: () => undefined,
+            done: () => complete,
+            next: async (amount) => {
+                const all = await getValues();
+                const batch = all.slice(offset, offset + amount);
+                offset += batch.length;
+                complete = offset >= all.length;
+                return coerce(batch);
+            },
+            all: async () => {
+                const all = await getValues();
+                const remaining = all.slice(offset);
+                offset = all.length;
+                complete = true;
+                return coerce(remaining);
+            },
+        };
+    }
     iterate(query, sort = this.properties.sort.sort, options) {
         let resolveInFull = options
             ? options === true
@@ -226,7 +464,7 @@ export class EntryIndex {
         };
         const coerce = async (results) => {
             if (resolveInFull) {
-                const maybeResolved = await Promise.all(results.map((x) => this.resolve(x.value.hash, resolveInFullOptions)));
+                const maybeResolved = await this.resolveMany(results.map((x) => x.value.hash), resolveInFullOptions);
                 return maybeResolved.filter((x) => !!x);
             }
             else {
@@ -241,17 +479,87 @@ export class EntryIndex {
         };
     }
     async getOldest(resolve) {
+        const nativeHash = this.getNativeTimestampOrderedHash("oldest");
+        if (nativeHash) {
+            const nativeResult = resolve
+                ? await this.resolve(nativeHash)
+                : (await this.getShallow(nativeHash))?.value;
+            if (nativeResult) {
+                return nativeResult;
+            }
+        }
         const iterator = this.iterate([], this.properties.sort.sort, resolve);
         const results = await iterator.next(1);
         await iterator.close();
         return results[0];
     }
+    async getOldestMany(limit, resolve) {
+        return this.getOldestManyMaybe(limit, resolve);
+    }
+    getOldestManyMaybe(limit, resolve) {
+        if (limit <= 0) {
+            return [];
+        }
+        const nativeEntries = this.getNativeOldestEntries(limit, resolve);
+        if (nativeEntries) {
+            return nativeEntries;
+        }
+        return this.getOldestManyFromIterator(limit, resolve);
+    }
+    async getOldestManyFromIterator(limit, resolve) {
+        const iterator = this.iterate([], this.properties.sort.sort, resolve);
+        try {
+            return (await iterator.next(limit));
+        }
+        finally {
+            await iterator.close();
+        }
+    }
+    getNativeOldestEntries(limit, resolve) {
+        if (resolve || limit <= 0) {
+            return;
+        }
+        const graph = this.properties.nativeGraph?.graph;
+        if (!graph?.oldestEntries) {
+            return;
+        }
+        const direction = timestampSortDirection(this.properties.sort.sort);
+        if (direction !== SortDirection.ASC) {
+            return;
+        }
+        return graph
+            .oldestEntries(limit)
+            .map((entry) => this.nativeLogEntryToShallowEntry(entry));
+    }
     async getNewest(resolve) {
+        const nativeHash = this.getNativeTimestampOrderedHash("newest");
+        if (nativeHash) {
+            const nativeResult = resolve
+                ? await this.resolve(nativeHash)
+                : (await this.getShallow(nativeHash))?.value;
+            if (nativeResult) {
+                return nativeResult;
+            }
+        }
         const iterator = this.iterate([], this.sortReversed, resolve);
         const results = await iterator.next(1);
         await iterator.close();
         return results[0];
     }
+    getNativeTimestampOrderedHash(position) {
+        const graph = this.properties.nativeGraph?.graph;
+        if (!graph?.oldestHash || !graph.newestHash) {
+            return;
+        }
+        const direction = timestampSortDirection(this.properties.sort.sort);
+        if (direction == null) {
+            return;
+        }
+        const useNewest = direction === SortDirection.ASC
+            ? position === "newest"
+            : position === "oldest";
+        return useNewest ? graph.newestHash() : graph.oldestHash();
+    }
     async getBefore(before, resolve) {
         const iterator = this.iterate(this.properties.sort.before(before), this.sortReversed, resolve);
         const results = await iterator.next(1);
@@ -267,17 +575,130 @@ export class EntryIndex {
     async get(k, options) {
         return this.resolve(k, options);
     }
+    async getMany(k, options) {
+        return this.resolveMany(k, options);
+    }
+    async planJoin(entry, reset) {
+        if (this.properties.nativeGraph) {
+            const cutCheck = this.properties.nativeGraph.useHeads
+                ? {
+                    gid: entry.meta.gid,
+                    wallTime: entry.meta.clock.timestamp.wallTime,
+                    logical: entry.meta.clock.timestamp.logical,
+                }
+                : undefined;
+            return this.properties.nativeGraph.graph.planJoin(entry.hash, entry.meta.next, entry.meta.type, reset === true, cutCheck);
+        }
+        if (!reset && (await this.getShallow(entry.hash)) != null) {
+            return {
+                skip: true,
+                missingParents: [],
+                cutChecked: false,
+                coveredByCut: false,
+            };
+        }
+        if (entry.meta.type === EntryType.CUT) {
+            return {
+                skip: false,
+                missingParents: [],
+                cutChecked: false,
+                coveredByCut: false,
+            };
+        }
+        const missingParents = [];
+        for (const next of entry.meta.next) {
+            if (reset || (await this.getShallow(next)) == null) {
+                missingParents.push(next);
+            }
+        }
+        return {
+            skip: false,
+            missingParents,
+            cutChecked: false,
+            coveredByCut: false,
+        };
+    }
+    async planJoinBatch(entries, reset, profile) {
+        if (entries.length === 0) {
+            return [];
+        }
+        if (this.properties.nativeGraph?.graph.planJoinBatch) {
+            const prepareStartedAt = entryIndexProfileStart(profile);
+            const nativeInputs = entries.map((entry) => ({
+                hash: entry.hash,
+                next: entry.meta.next,
+                type: entry.meta.type,
+                cutCheck: this.properties.nativeGraph.useHeads
+                    ? {
+                        gid: entry.meta.gid,
+                        wallTime: entry.meta.clock.timestamp.wallTime,
+                        logical: entry.meta.clock.timestamp.logical,
+                    }
+                    : undefined,
+            }));
+            emitEntryIndexProfileDuration(profile, prepareStartedAt, {
+                name: "log.entryIndex.planJoinBatch.prepareNative",
+                component: "log",
+                entries: entries.length,
+                messages: 1,
+                details: {
+                    cutChecks: nativeInputs.reduce((sum, input) => sum + (input.cutCheck ? 1 : 0), 0),
+                },
+            });
+            const nativeStartedAt = entryIndexProfileStart(profile);
+            const plans = this.properties.nativeGraph.graph.planJoinBatch(nativeInputs, reset === true);
+            emitEntryIndexProfileDuration(profile, nativeStartedAt, {
+                name: "log.entryIndex.planJoinBatch.nativeGraph",
+                component: "log",
+                entries: entries.length,
+                messages: 1,
+            });
+            return plans;
+        }
+        const fallbackStartedAt = entryIndexProfileStart(profile);
+        const plans = [];
+        for (const entry of entries) {
+            plans.push(await this.planJoin(entry, reset));
+        }
+        emitEntryIndexProfileDuration(profile, fallbackStartedAt, {
+            name: "log.entryIndex.planJoinBatch.fallback",
+            component: "log",
+            entries: entries.length,
+            messages: 1,
+        });
+        return plans;
+    }
     async getShallow(k) {
-        const pending = this.pendingIndexWrites.get(k);
+        const pending = this.getPendingIndexWrite(k);
         if (pending) {
             return { id: toId(k), value: pending };
         }
         return this.properties.index.get(toId(k));
     }
+    getNativeEntryMetadataBatch(hashes) {
+        if (!this.properties.nativeGraph) {
+            return undefined;
+        }
+        const normalized = [...hashes];
+        if (normalized.length === 0) {
+            return [];
+        }
+        return this.properties.nativeGraph.graph.entryMetadataBatch?.(normalized);
+    }
+    getNativeEntryMetadataHintsBatch(hashes) {
+        if (!this.properties.nativeGraph) {
+            return undefined;
+        }
+        const normalized = [...hashes];
+        if (normalized.length === 0) {
+            return [];
+        }
+        return (this.properties.nativeGraph.graph.entryMetadataHintsBatch?.(normalized) ??
+            this.properties.nativeGraph.graph.entryMetadataBatch?.(normalized));
+    }
     async has(k) {
-        let mem = this.cache.get(k);
-        if (mem) {
-            return true;
+        if (this.properties.nativeGraph) {
+            return this.properties.nativeGraph.graph.has(k);
         }
         if (this.pendingIndexWrites.has(k)) {
             return true;
@@ -288,11 +709,14 @@ export class EntryIndex {
         return result != null;
     }
     async hasMany(hashes) {
+        if (this.properties.nativeGraph) {
+            return this.properties.nativeGraph.graph.hasMany(new Set([...hashes].filter(Boolean)));
+        }
         const batchSize = 64;
         const existing = new Set();
         const missing = [];
         for (const hash of new Set([...hashes].filter(Boolean))) {
-            if (this.cache.get(hash) || this.pendingIndexWrites.has(hash)) {
+            if (this.pendingIndexWrites.has(hash)) {
                 existing.add(hash);
                 continue;
             }
@@ -331,9 +755,17 @@ export class EntryIndex {
     async put(entry, properties) {
         if (properties.toMultiHash) {
             const existingHash = entry.hash;
-            entry.hash = undefined;
+            const preparedBlock = Entry.takePreparedBlock(entry);
             try {
-                const hash = await Entry.toMultihash(this.properties.store, entry);
+                let hash;
+                if (preparedBlock) {
+                    hash = await putPreparedEntryBlock(this.properties.store, preparedBlock);
+                    entry.size = preparedBlock.block.bytes.length;
+                }
+                else {
+                    entry.hash = undefined;
+                    hash = await Entry.toMultihash(this.properties.store, entry);
+                }
                 entry.hash = existingHash;
                 if (entry.hash === undefined) {
                     entry.hash = hash; // can happen if you sync entries that you load directly from ipfs
@@ -364,7 +796,10 @@ export class EntryIndex {
                 }
                 // add cache after .has check before .has uses the cache
                 this.cache.add(entry.hash, entry);
-                const shallowEntry = entry.toShallow(properties.isHead);
+                const shallowEntry = Entry.takePreparedShallowEntry(entry, properties.isHead) ??
+                    entry.toShallow(properties.isHead);
+                const nativeEntry = Entry.takePreparedNativeLogEntry(entry, properties.isHead) ??
+                    toNativeLogEntry(shallowEntry);
                 const shouldDeferIndexWrite = properties.deferIndexWrite === true &&
                     properties.isHead &&
                     entry.meta.type !== EntryType.CUT &&
@@ -377,16 +812,9 @@ export class EntryIndex {
                     await this.flushPendingWrites(entry.meta.next);
                     await this.properties.index.put(shallowEntry);
                 }
-                this.properties.nativeGraph?.graph.put(toNativeLogEntry(shallowEntry));
+                this.properties.nativeGraph?.graph.put(nativeEntry);
                 // check if gids has been shadowed, by query all nexts that have a different gid
-                if (this.properties.onGidRemoved && entry.meta.next.length > 0) {
-                    const shadowedGids = this.properties.nativeGraph
-                        ? new Set(this.properties.nativeGraph.graph.shadowedGids(entry.meta.gid, entry.meta.next, entry.hash))
-                        : await this.findShadowedGids(entry);
-                    if (shadowedGids.size > 0) {
-                        this.properties.onGidRemoved?.([...shadowedGids]);
-                    }
-                }
+                await this.notifyShadowedGids(entry);
                 // mark all next entries as not heads
                 await this.privateUpdateNextHeadProperty(entry, false);
                 this.insertionPromises.delete(entry.hash);
@@ -396,12 +824,560 @@ export class EntryIndex {
             return promise;
         }
     }
+    async putAppendBatch(entries, properties) {
+        if (entries.length === 0) {
+            return;
+        }
+        if (entries.length === 1 &&
+            properties.prepared?.nativeGraphUpdated !== true) {
+            return this.put(entries[0], {
+                unique: properties.unique,
+                isHead: true,
+                toMultiHash: false,
+                deferIndexWrite: properties.deferIndexWrite,
+            });
+        }
+        for (const entry of entries) {
+            if (!entry.hash) {
+                throw new Error("Missing hash");
+            }
+            const existingPromise = this.insertionPromises.get(entry.hash);
+            if (existingPromise) {
+                await existingPromise;
+            }
+        }
+        const promise = (async () => {
+            const profile = properties.profile;
+            const prepareStartedAt = entryIndexProfileStart(profile);
+            const externalNexts = new Set(properties.externalNextHashes ?? []);
+            const shouldDiscoverExternalNexts = !properties.externalNextHashes;
+            const batchHashes = shouldDiscoverExternalNexts
+                ? new Set(entries.map((entry) => entry.hash))
+                : undefined;
+            const putBatch = !this.properties.onGidRemoved && this.properties.index.putBatch;
+            const shallowEntries = [];
+            const nativeGraphUpdated = properties.prepared?.nativeGraphUpdated === true;
+            const nativeCommitOwnsHotIndex = nativeGraphUpdated &&
+                properties.prepared?.nativeBlocksCommitted === true &&
+                !this.properties.onGidRemoved;
+            const nativeGraphPutAppendChain = !nativeGraphUpdated &&
+                !this.properties.onGidRemoved &&
+                properties.externalNextHashes &&
+                this.properties.nativeGraph?.graph.putAppendChain
+                ? this.properties.nativeGraph.graph.putAppendChain.bind(this.properties.nativeGraph.graph)
+                : undefined;
+            const nativeGraphPutBatch = !nativeGraphUpdated &&
+                !this.properties.onGidRemoved &&
+                this.properties.nativeGraph?.graph.putBatch
+                ? this.properties.nativeGraph.graph.putBatch.bind(this.properties.nativeGraph.graph)
+                : undefined;
+            const deferBatchIndexWrite = properties.deferIndexWrite === true &&
+                !!putBatch &&
+                !!this.properties.nativeGraph &&
+                !this.properties.onGidRemoved &&
+                entries.every((entry) => entry.meta.type !== EntryType.CUT);
+            const nativeEntries = [];
+            for (let i = 0; i < entries.length; i++) {
+                const entry = entries[i];
+                const isHead = properties.heads?.[i] ?? i === entries.length - 1;
+                if (properties.unique === true || !(await this.has(entry.hash))) {
+                    this._length++;
+                }
+                this.cache.add(entry.hash, entry);
+                const preparedShallowEntry = properties.prepared?.shallowEntries[i];
+                if (preparedShallowEntry) {
+                    preparedShallowEntry.head = isHead;
+                }
+                const shallowEntry = preparedShallowEntry ??
+                    Entry.takePreparedShallowEntry(entry, isHead) ??
+                    entry.toShallow(isHead);
+                if (nativeCommitOwnsHotIndex) {
+                    this.pendingIndexWrites.set(entry.hash, shallowEntry);
+                    if (batchHashes) {
+                        for (const next of entry.meta.next) {
+                            if (!batchHashes.has(next)) {
+                                externalNexts.add(next);
+                            }
+                        }
+                    }
+                    continue;
+                }
+                const preparedNativeEntry = properties.prepared?.nativeEntries?.[i];
+                if (preparedNativeEntry) {
+                    preparedNativeEntry.head = isHead;
+                }
+                const nativeEntry = !nativeGraphUpdated &&
+                    this.properties.nativeGraph &&
+                    (preparedNativeEntry ??
+                        Entry.takePreparedNativeLogEntry(entry, isHead) ??
+                        toNativeLogEntry(shallowEntry));
+                if (putBatch) {
+                    shallowEntries.push(shallowEntry);
+                    nativeEntry && nativeEntries.push(nativeEntry);
+                }
+                else {
+                    await this.properties.index.put(shallowEntry);
+                    if (nativeGraphPutAppendChain || nativeGraphPutBatch) {
+                        nativeEntry && nativeEntries.push(nativeEntry);
+                    }
+                    else {
+                        if (nativeEntry) {
+                            this.properties.nativeGraph?.graph.put(nativeEntry);
+                            await this.notifyShadowedGids(entry);
+                        }
+                    }
+                }
+                if (batchHashes) {
+                    for (const next of entry.meta.next) {
+                        if (!batchHashes.has(next)) {
+                            externalNexts.add(next);
+                        }
+                    }
+                }
+            }
+            emitEntryIndexProfileDuration(profile, prepareStartedAt, {
+                name: "log.entryIndex.putAppendBatch.prepare",
+                component: "log",
+                entries: entries.length,
+                messages: 1,
+                details: {
+                    unique: properties.unique,
+                    usedPreparedShallowEntries: properties.prepared?.shallowEntries.length ?? 0,
+                    usedPreparedNativeEntries: properties.prepared?.nativeEntries?.length ?? 0,
+                    nativeEntries: nativeEntries.length,
+                    discoverExternalNexts: shouldDiscoverExternalNexts,
+                },
+            });
+            const putNativeEntries = (allowLoopFallback) => {
+                if (nativeEntries.length === 0) {
+                    return;
+                }
+                if (nativeGraphPutAppendChain) {
+                    const nativePutStartedAt = entryIndexProfileStart(profile);
+                    nativeGraphPutAppendChain(nativeEntries);
+                    emitEntryIndexProfileDuration(profile, nativePutStartedAt, {
+                        name: "log.entryIndex.putAppendBatch.nativeGraphPut",
+                        component: "log",
+                        entries: nativeEntries.length,
+                        messages: 1,
+                        details: { method: "putAppendChain" },
+                    });
+                }
+                else if (nativeGraphPutBatch) {
+                    const nativePutStartedAt = entryIndexProfileStart(profile);
+                    nativeGraphPutBatch(nativeEntries);
+                    emitEntryIndexProfileDuration(profile, nativePutStartedAt, {
+                        name: "log.entryIndex.putAppendBatch.nativeGraphPut",
+                        component: "log",
+                        entries: nativeEntries.length,
+                        messages: 1,
+                        details: { method: "putBatch" },
+                    });
+                }
+                else if (allowLoopFallback) {
+                    const nativePutStartedAt = entryIndexProfileStart(profile);
+                    for (const nativeEntry of nativeEntries) {
+                        this.properties.nativeGraph?.graph.put(nativeEntry);
+                    }
+                    emitEntryIndexProfileDuration(profile, nativePutStartedAt, {
+                        name: "log.entryIndex.putAppendBatch.nativeGraphPut",
+                        component: "log",
+                        entries: nativeEntries.length,
+                        messages: 1,
+                        details: { method: "putLoop" },
+                    });
+                }
+            };
+            if (nativeCommitOwnsHotIndex) {
+                this.schedulePendingIndexWriteFlush();
+            }
+            else if (deferBatchIndexWrite) {
+                const indexPutStartedAt = entryIndexProfileStart(profile);
+                for (const shallowEntry of shallowEntries) {
+                    this.pendingIndexWrites.set(shallowEntry.hash, shallowEntry);
+                }
+                this.schedulePendingIndexWriteFlush();
+                emitEntryIndexProfileDuration(profile, indexPutStartedAt, {
+                    name: "log.entryIndex.putAppendBatch.indexPut",
+                    component: "log",
+                    entries: shallowEntries.length,
+                    messages: 1,
+                    details: { deferred: true },
+                });
+                putNativeEntries(true);
+            }
+            else if (putBatch) {
+                const indexPutStartedAt = entryIndexProfileStart(profile);
+                await putBatch.call(this.properties.index, shallowEntries);
+                emitEntryIndexProfileDuration(profile, indexPutStartedAt, {
+                    name: "log.entryIndex.putAppendBatch.indexPut",
+                    component: "log",
+                    entries: shallowEntries.length,
+                    messages: 1,
+                });
+                putNativeEntries(true);
+            }
+            else if (nativeEntries.length > 0) {
+                putNativeEntries(false);
+            }
+            if (externalNexts.size > 0) {
+                const externalNextStartedAt = entryIndexProfileStart(profile);
+                await this.privateUpdateNextHeadHashes([...externalNexts], false);
+                emitEntryIndexProfileDuration(profile, externalNextStartedAt, {
+                    name: "log.entryIndex.putAppendBatch.externalNexts",
+                    component: "log",
+                    entries: externalNexts.size,
+                    messages: 1,
+                });
+            }
+        })().finally(() => {
+            for (const entry of entries) {
+                this.insertionPromises.delete(entry.hash);
+            }
+        });
+        for (const entry of entries) {
+            this.insertionPromises.set(entry.hash, promise);
+        }
+        return promise;
+    }
+    // Internal trusted receive path for callers that can supply prepared append facts.
+    async putAppendFactsBatch(entries, properties) {
+        if (entries.length === 0) {
+            return;
+        }
+        if (this.properties.onGidRemoved) {
+            throw new Error("Prepared append facts batch requires no onGidRemoved hook");
+        }
+        for (const entry of entries) {
+            if (!entry.hash) {
+                throw new Error("Missing hash");
+            }
+            const existingPromise = this.insertionPromises.get(entry.hash);
+            if (existingPromise) {
+                await existingPromise;
+            }
+        }
+        const promise = (async () => {
+            const profile = properties.profile;
+            const prepareStartedAt = entryIndexProfileStart(profile);
+            const externalNexts = new Set(properties.externalNextHashes ?? []);
+            const shouldDiscoverExternalNexts = !properties.externalNextHashes;
+            const batchHashes = shouldDiscoverExternalNexts
+                ? new Set(entries.map((entry) => entry.hash))
+                : undefined;
+            const putBatch = this.properties.index.putBatch;
+            const shallowEntries = [];
+            const nativeEntries = [];
+            const nativeGraphUpdated = properties.nativeGraphUpdated === true;
+            const nativeGraphPutAppendChain = !nativeGraphUpdated &&
+                properties.externalNextHashes &&
+                this.properties.nativeGraph?.graph.putAppendChain
+                ? this.properties.nativeGraph.graph.putAppendChain.bind(this.properties.nativeGraph.graph)
+                : undefined;
+            const nativeGraphPutBatch = !nativeGraphUpdated && this.properties.nativeGraph?.graph.putBatch
+                ? this.properties.nativeGraph.graph.putBatch.bind(this.properties.nativeGraph.graph)
+                : undefined;
+            const deferBatchIndexWrite = properties.deferIndexWrite === true &&
+                !!this.properties.nativeGraph &&
+                entries.every((entry) => entry.meta.type !== EntryType.CUT);
+            const lazyNativeGraphUpdatedDeferredIndexWrite = nativeGraphUpdated && deferBatchIndexWrite;
+            for (let i = 0; i < entries.length; i++) {
+                const entry = entries[i];
+                const isHead = properties.heads?.[i] ?? i === entries.length - 1;
+                if (properties.unique === true || !(await this.has(entry.hash))) {
+                    this._length++;
+                }
+                let shallowEntry;
+                if (lazyNativeGraphUpdatedDeferredIndexWrite) {
+                    this.pendingIndexWrites.set(entry.hash, () => materializePreparedAppendShallowEntry(entry, isHead));
+                }
+                else {
+                    shallowEntry = materializePreparedAppendShallowEntry(entry, isHead);
+                }
+                const nativeEntry = !nativeGraphUpdated &&
+                    this.properties.nativeGraph &&
+                    (entry.nativeEntry ??
+                        toNativeLogEntry(shallowEntry ??
+                            materializePreparedAppendShallowEntry(entry, isHead)));
+                if (nativeEntry) {
+                    nativeEntry.head = isHead;
+                }
+                if (lazyNativeGraphUpdatedDeferredIndexWrite) {
+                    // The native commit already updated blocks/graph; keep the JS
+                    // compatibility index write lazy for later flushes.
+                }
+                else if (deferBatchIndexWrite || putBatch) {
+                    shallowEntries.push(shallowEntry);
+                    nativeEntry && nativeEntries.push(nativeEntry);
+                }
+                else {
+                    await this.properties.index.put(shallowEntry);
+                    nativeEntry && nativeEntries.push(nativeEntry);
+                }
+                if (batchHashes) {
+                    for (const next of entry.meta.next) {
+                        if (!batchHashes.has(next)) {
+                            externalNexts.add(next);
+                        }
+                    }
+                }
+            }
+            emitEntryIndexProfileDuration(profile, prepareStartedAt, {
+                name: "log.entryIndex.putAppendFactsBatch.prepare",
+                component: "log",
+                entries: entries.length,
+                messages: 1,
+                details: {
+                    unique: properties.unique,
+                    nativeEntries: nativeEntries.length,
+                    discoverExternalNexts: shouldDiscoverExternalNexts,
+                    nativeGraphUpdated,
+                },
+            });
+            const putNativeEntries = (allowLoopFallback) => {
+                if (nativeEntries.length === 0) {
+                    return;
+                }
+                if (nativeGraphPutAppendChain) {
+                    const nativePutStartedAt = entryIndexProfileStart(profile);
+                    nativeGraphPutAppendChain(nativeEntries);
+                    emitEntryIndexProfileDuration(profile, nativePutStartedAt, {
+                        name: "log.entryIndex.putAppendFactsBatch.nativeGraphPut",
+                        component: "log",
+                        entries: nativeEntries.length,
+                        messages: 1,
+                        details: { method: "putAppendChain" },
+                    });
+                }
+                else if (nativeGraphPutBatch) {
+                    const nativePutStartedAt = entryIndexProfileStart(profile);
+                    nativeGraphPutBatch(nativeEntries);
+                    emitEntryIndexProfileDuration(profile, nativePutStartedAt, {
+                        name: "log.entryIndex.putAppendFactsBatch.nativeGraphPut",
+                        component: "log",
+                        entries: nativeEntries.length,
+                        messages: 1,
+                        details: { method: "putBatch" },
+                    });
+                }
+                else if (allowLoopFallback) {
+                    const nativePutStartedAt = entryIndexProfileStart(profile);
+                    for (const nativeEntry of nativeEntries) {
+                        this.properties.nativeGraph?.graph.put(nativeEntry);
+                    }
+                    emitEntryIndexProfileDuration(profile, nativePutStartedAt, {
+                        name: "log.entryIndex.putAppendFactsBatch.nativeGraphPut",
+                        component: "log",
+                        entries: nativeEntries.length,
+                        messages: 1,
+                        details: { method: "putLoop" },
+                    });
+                }
+            };
+            if (deferBatchIndexWrite) {
+                const indexPutStartedAt = entryIndexProfileStart(profile);
+                if (!lazyNativeGraphUpdatedDeferredIndexWrite) {
+                    for (const shallowEntry of shallowEntries) {
+                        this.pendingIndexWrites.set(shallowEntry.hash, shallowEntry);
+                    }
+                }
+                this.schedulePendingIndexWriteFlush();
+                emitEntryIndexProfileDuration(profile, indexPutStartedAt, {
+                    name: "log.entryIndex.putAppendFactsBatch.indexPut",
+                    component: "log",
+                    entries: lazyNativeGraphUpdatedDeferredIndexWrite
+                        ? entries.length
+                        : shallowEntries.length,
+                    messages: 1,
+                    details: {
+                        deferred: true,
+                        lazy: lazyNativeGraphUpdatedDeferredIndexWrite,
+                    },
+                });
+                putNativeEntries(true);
+            }
+            else if (putBatch) {
+                const indexPutStartedAt = entryIndexProfileStart(profile);
+                await putBatch.call(this.properties.index, shallowEntries);
+                emitEntryIndexProfileDuration(profile, indexPutStartedAt, {
+                    name: "log.entryIndex.putAppendFactsBatch.indexPut",
+                    component: "log",
+                    entries: shallowEntries.length,
+                    messages: 1,
+                });
+                putNativeEntries(true);
+            }
+            else if (nativeEntries.length > 0) {
+                putNativeEntries(true);
+            }
+            if (externalNexts.size > 0) {
+                const externalNextStartedAt = entryIndexProfileStart(profile);
+                await this.privateUpdateNextHeadHashes([...externalNexts], false);
+                emitEntryIndexProfileDuration(profile, externalNextStartedAt, {
+                    name: "log.entryIndex.putAppendFactsBatch.externalNexts",
+                    component: "log",
+                    entries: externalNexts.size,
+                    messages: 1,
+                });
+            }
+        })().finally(() => {
+            for (const entry of entries) {
+                this.insertionPromises.delete(entry.hash);
+            }
+        });
+        for (const entry of entries) {
+            this.insertionPromises.set(entry.hash, promise);
+        }
+        return promise;
+    }
+    /** @internal */
+    async putNativeCommittedAppend(entry, properties) {
+        if (!entry.hash) {
+            throw new Error("Missing hash");
+        }
+        const existingPromise = this.insertionPromises.get(entry.hash);
+        if (existingPromise) {
+            await existingPromise;
+        }
+        const promise = (async () => {
+            const isHead = properties.isHead ?? true;
+            if (properties.unique === true || !(await this.has(entry.hash))) {
+                this._length++;
+            }
+            this.cache.add(entry.hash, entry);
+            const shallowEntry = properties.shallowEntry ??
+                Entry.takePreparedShallowEntry(entry, isHead) ??
+                entry.toShallow(isHead);
+            shallowEntry.head = isHead;
+            this.pendingIndexWrites.set(entry.hash, shallowEntry);
+            this.schedulePendingIndexWriteFlush();
+            if (properties.externalNextHashes.length > 0) {
+                await this.privateUpdateNextHeadHashes(properties.externalNextHashes, false);
+            }
+        })().finally(() => {
+            this.insertionPromises.delete(entry.hash);
+        });
+        this.insertionPromises.set(entry.hash, promise);
+        return promise;
+    }
+    /** @internal */
+    putNativeCommittedAppendFacts(properties) {
+        if (!properties.hash) {
+            throw new Error("Missing hash");
+        }
+        const existingPromise = this.insertionPromises.get(properties.hash);
+        if (!existingPromise &&
+            properties.unique === true &&
+            properties.externalNextHashes.length === 0) {
+            const isHead = properties.isHead ?? true;
+            this._length++;
+            if (!properties.shallowEntry && !properties.getShallowEntry) {
+                throw new Error("Missing shallow entry");
+            }
+            const pending = properties.shallowEntry ??
+                (() => {
+                    const shallowEntry = properties.getShallowEntry();
+                    shallowEntry.head = isHead;
+                    return shallowEntry;
+                });
+            if (properties.shallowEntry) {
+                properties.shallowEntry.head = isHead;
+            }
+            this.pendingIndexWrites.set(properties.hash, pending);
+            this.schedulePendingIndexWriteFlush();
+            return;
+        }
+        return this.putNativeCommittedAppendFactsAsync(properties, existingPromise);
+    }
+    /** @internal */
+    putNativeCommittedAppendFactsBatch(rows) {
+        if (rows.length === 0) {
+            return;
+        }
+        const asyncRows = [];
+        const externalNextHashes = [];
+        let scheduledPendingFlush = false;
+        for (const row of rows) {
+            if (!row.hash) {
+                throw new Error("Missing hash");
+            }
+            const existingPromise = this.insertionPromises.get(row.hash);
+            if (!existingPromise && row.unique === true) {
+                const isHead = row.isHead ?? true;
+                this._length++;
+                if (!row.shallowEntry && !row.getShallowEntry) {
+                    throw new Error("Missing shallow entry");
+                }
+                const pending = row.shallowEntry ??
+                    (() => {
+                        const shallowEntry = row.getShallowEntry();
+                        shallowEntry.head = isHead;
+                        return shallowEntry;
+                    });
+                if (row.shallowEntry) {
+                    row.shallowEntry.head = isHead;
+                }
+                this.pendingIndexWrites.set(row.hash, pending);
+                if (row.externalNextHashes.length > 0) {
+                    externalNextHashes.push(...row.externalNextHashes);
+                }
+                scheduledPendingFlush = true;
+            }
+            else {
+                asyncRows.push(row);
+            }
+        }
+        if (scheduledPendingFlush) {
+            this.schedulePendingIndexWriteFlush();
+        }
+        const batchedNextUpdate = externalNextHashes.length > 0
+            ? this.privateUpdateNextHeadHashes(externalNextHashes, false)
+            : undefined;
+        if (asyncRows.length > 0 || batchedNextUpdate) {
+            return Promise.all([
+                batchedNextUpdate,
+                ...asyncRows.map((row) => this.putNativeCommittedAppendFacts(row)),
+            ]).then(() => undefined);
+        }
+    }
+    async putNativeCommittedAppendFactsAsync(properties, existingPromise) {
+        if (existingPromise) {
+            await existingPromise;
+        }
+        const promise = (async () => {
+            const isHead = properties.isHead ?? true;
+            if (properties.unique === true || !(await this.has(properties.hash))) {
+                this._length++;
+            }
+            if (!properties.shallowEntry && !properties.getShallowEntry) {
+                throw new Error("Missing shallow entry");
+            }
+            const pending = properties.shallowEntry ??
+                (() => {
+                    const shallowEntry = properties.getShallowEntry();
+                    shallowEntry.head = isHead;
+                    return shallowEntry;
+                });
+            if (properties.shallowEntry) {
+                properties.shallowEntry.head = isHead;
+            }
+            this.pendingIndexWrites.set(properties.hash, pending);
+            this.schedulePendingIndexWriteFlush();
+            if (properties.externalNextHashes.length > 0) {
+                await this.privateUpdateNextHeadHashes(properties.externalNextHashes, false);
+            }
+        })().finally(() => {
+            this.insertionPromises.delete(properties.hash);
+        });
+        this.insertionPromises.set(properties.hash, promise);
+        return promise;
+    }
     async delete(k, from) {
         this.cache.del(k);
         if (from && from.hash !== k) {
             throw new Error("Shallow hash doesn't match the key");
         }
-        const pending = this.pendingIndexWrites.get(k);
+        const pending = this.getPendingIndexWrite(k);
         from = from || pending || (await this.getShallow(k))?.value;
         if (!from) {
             return; // already deleted
@@ -424,18 +1400,397 @@ export class EntryIndex {
             return from;
         }
     }
+    canDeleteMany() {
+        return (!!this.properties.nativeGraph?.graph.deleteMany ||
+            hasRmMany(this.properties.store));
+    }
+    async deleteMany(from, options) {
+        return this.deleteManyMaybe(from, options);
+    }
+    deleteManyMaybe(from, options) {
+        if (from.length === 0) {
+            return [];
+        }
+        if (from.length === 1) {
+            return this.deleteSingleMaybe(from[0], options);
+        }
+        const nodes = [];
+        const seen = new Set();
+        for (const node of from) {
+            if (seen.has(node.hash)) {
+                continue;
+            }
+            seen.add(node.hash);
+            nodes.push(node);
+        }
+        const indexedByHash = new Map(nodes.map((node) => [node.hash, node]));
+        const deletedByHash = new Map();
+        const indexedHashes = [];
+        const storeHashes = [];
+        for (const node of nodes) {
+            this.cache.del(node.hash);
+            const pending = this.getPendingIndexWrite(node.hash);
+            if (pending) {
+                this.pendingIndexWrites.delete(node.hash);
+                deletedByHash.set(node.hash, pending);
+                storeHashes.push(node.hash);
+            }
+            else {
+                indexedHashes.push(node.hash);
+            }
+        }
+        const exactDeleteIndex = hasExactDelete(this.properties.index)
+            ? this.properties.index
+            : undefined;
+        if (indexedHashes.length > 0 && exactDeleteIndex) {
+            return mapMaybePromise(exactDeleteIndex.delIds(indexedHashes), (deleted) => {
+                for (const id of deleted) {
+                    const hash = String(id.primitive);
+                    const node = indexedByHash.get(hash);
+                    if (!node || deletedByHash.has(hash)) {
+                        continue;
+                    }
+                    deletedByHash.set(hash, node);
+                    storeHashes.push(hash);
+                }
+                return this.finishDeleteMany(deletedByHash, nodes, storeHashes, options);
+            });
+        }
+        if (indexedHashes.length > 0) {
+            return this.deleteManyByQuery(indexedHashes, indexedByHash, deletedByHash, nodes, storeHashes, options);
+        }
+        return this.finishDeleteMany(deletedByHash, nodes, storeHashes, options);
+    }
+    consumeNativeTrimmedEntriesMaybe(from, options) {
+        if (from.length === 0) {
+            return [];
+        }
+        const nodes = [];
+        const seen = new Set();
+        for (const node of from) {
+            if (seen.has(node.hash)) {
+                continue;
+            }
+            seen.add(node.hash);
+            nodes.push(node);
+        }
+        return this.consumeNativeTrimmedEntryNodesMaybe(nodes, options);
+    }
+    consumeNativeTrimmedEntryHashesMaybe(hashes, options) {
+        if (hashes.length === 0) {
+            return [];
+        }
+        const nodes = [];
+        const seen = new Set();
+        for (const hash of hashes) {
+            if (seen.has(hash)) {
+                continue;
+            }
+            seen.add(hash);
+            nodes.push(this.nativeTrimmedHashToShallowEntry(hash));
+        }
+        return this.consumeNativeTrimmedEntryNodesMaybe(nodes, options);
+    }
+    consumeNativeTrimmedEntryHashesNoReturnMaybe(hashes, options) {
+        if (!options?.skipNextHeadUpdates ||
+            options.deleteBlocks !== false ||
+            hashes.length === 0) {
+            return undefined;
+        }
+        const indexedHashes = [];
+        let pendingDeleted = 0;
+        const seen = new Set();
+        for (const hash of hashes) {
+            if (seen.has(hash)) {
+                continue;
+            }
+            seen.add(hash);
+            this.cache.del(hash);
+            if (this.pendingIndexWrites.delete(hash)) {
+                pendingDeleted++;
+            }
+            else {
+                indexedHashes.push(hash);
+            }
+        }
+        const finish = (indexedDeleted) => {
+            this._length -= pendingDeleted + indexedDeleted;
+            return true;
+        };
+        if (indexedHashes.length === 0) {
+            return finish(0);
+        }
+        const exactDeleteCountIndex = hasExactDeleteCount(this.properties.index)
+            ? this.properties.index
+            : undefined;
+        if (exactDeleteCountIndex) {
+            return mapMaybePromise(exactDeleteCountIndex.delIdsCount(indexedHashes), finish);
+        }
+        const exactDeleteIndex = hasExactDelete(this.properties.index)
+            ? this.properties.index
+            : undefined;
+        if (exactDeleteIndex) {
+            return mapMaybePromise(exactDeleteIndex.delIds(indexedHashes), (deleted) => finish(deleted.length));
+        }
+        return this.consumeNativeTrimmedEntryHashesNoReturnByQuery(indexedHashes, finish);
+    }
+    async consumeNativeTrimmedEntryHashesNoReturnByQuery(indexedHashes, finish) {
+        const batchSize = 64;
+        let deletedCount = 0;
+        for (let i = 0; i < indexedHashes.length; i += batchSize) {
+            const hashes = indexedHashes.slice(i, i + batchSize);
+            const deleted = await this.properties.index.del({
+                query: createHashMatchQuery(hashes),
+            });
+            deletedCount += deleted.length;
+        }
+        return finish(deletedCount);
+    }
+    consumeNativeTrimmedEntryNodesMaybe(nodes, options) {
+        const indexedByHash = new Map(nodes.map((node) => [node.hash, node]));
+        const deletedByHash = new Map();
+        const indexedHashes = [];
+        for (const node of nodes) {
+            this.cache.del(node.hash);
+            const pending = this.getPendingIndexWrite(node.hash);
+            if (pending) {
+                this.pendingIndexWrites.delete(node.hash);
+                deletedByHash.set(node.hash, pending);
+            }
+            else {
+                indexedHashes.push(node.hash);
+            }
+        }
+        const finish = () => this.finishConsumeNativeTrimmedEntries(deletedByHash, nodes, options);
+        const exactDeleteIndex = hasExactDelete(this.properties.index)
+            ? this.properties.index
+            : undefined;
+        if (indexedHashes.length > 0 && exactDeleteIndex) {
+            return mapMaybePromise(exactDeleteIndex.delIds(indexedHashes), (deleted) => {
+                for (const id of deleted) {
+                    const hash = String(id.primitive);
+                    const node = indexedByHash.get(hash);
+                    if (!node || deletedByHash.has(hash)) {
+                        continue;
+                    }
+                    deletedByHash.set(hash, node);
+                }
+                return finish();
+            });
+        }
+        if (indexedHashes.length > 0) {
+            return this.consumeNativeTrimmedEntriesByQuery(indexedHashes, indexedByHash, deletedByHash, nodes, options);
+        }
+        return finish();
+    }
+    deleteSingleMaybe(node, options) {
+        this.cache.del(node.hash);
+        const pending = this.getPendingIndexWrite(node.hash);
+        if (pending) {
+            this.pendingIndexWrites.delete(node.hash);
+            return this.finishDeleteSingle(pending, options);
+        }
+        const exactDeleteIndex = hasExactDelete(this.properties.index)
+            ? this.properties.index
+            : undefined;
+        if (exactDeleteIndex) {
+            return mapMaybePromise(exactDeleteIndex.delIds([node.hash]), (deleted) => deleted.some((id) => String(id.primitive) === node.hash)
+                ? this.finishDeleteSingle(node, options)
+                : []);
+        }
+        return mapMaybePromise(this.properties.index.del({ query: createHashMatchQuery([node.hash]) }), (deleted) => deleted.some((id) => String(id.primitive) === node.hash)
+            ? this.finishDeleteSingle(node, options)
+            : []);
+    }
+    finishDeleteSingle(node, options) {
+        const afterStoreDelete = () => {
+            this._length--;
+            this.properties.nativeGraph?.graph.delete(node.hash);
+            if (!options?.skipNextHeadUpdates && node.meta.type !== EntryType.CUT) {
+                return mapMaybePromise(this.privateUpdateNextHeadHashes(node.meta.next, true), () => [node]);
+            }
+            return [node];
+        };
+        return mapMaybePromise(this.properties.store.rm(node.hash), afterStoreDelete);
+    }
+    async deleteManyByQuery(indexedHashes, indexedByHash, deletedByHash, nodes, storeHashes, options) {
+        const batchSize = 64;
+        for (let i = 0; i < indexedHashes.length; i += batchSize) {
+            const hashes = indexedHashes.slice(i, i + batchSize);
+            const deleted = await this.properties.index.del({
+                query: createHashMatchQuery(hashes),
+            });
+            for (const id of deleted) {
+                const hash = String(id.primitive);
+                const node = indexedByHash.get(hash);
+                if (!node || deletedByHash.has(hash)) {
+                    continue;
+                }
+                deletedByHash.set(hash, node);
+                storeHashes.push(hash);
+            }
+        }
+        return this.finishDeleteMany(deletedByHash, nodes, storeHashes, options);
+    }
+    async consumeNativeTrimmedEntriesByQuery(indexedHashes, indexedByHash, deletedByHash, nodes, options) {
+        const batchSize = 64;
+        for (let i = 0; i < indexedHashes.length; i += batchSize) {
+            const hashes = indexedHashes.slice(i, i + batchSize);
+            const deleted = await this.properties.index.del({
+                query: createHashMatchQuery(hashes),
+            });
+            for (const id of deleted) {
+                const hash = String(id.primitive);
+                const node = indexedByHash.get(hash);
+                if (!node || deletedByHash.has(hash)) {
+                    continue;
+                }
+                deletedByHash.set(hash, node);
+            }
+        }
+        return this.finishConsumeNativeTrimmedEntries(deletedByHash, nodes, options);
+    }
+    finishConsumeNativeTrimmedEntries(deletedByHash, nodes, options) {
+        const deleted = nodes
+            .map((node) => deletedByHash.get(node.hash))
+            .filter((node) => !!node);
+        if (deleted.length === 0) {
+            return [];
+        }
+        const afterStoreDelete = () => {
+            this._length -= deleted.length;
+            if (!options?.skipNextHeadUpdates) {
+                const deletedNexts = [];
+                for (const node of deleted) {
+                    if (node.meta.type !== EntryType.CUT) {
+                        deletedNexts.push(...node.meta.next);
+                    }
+                }
+                return mapMaybePromise(this.privateUpdateNextHeadHashes(deletedNexts, true), () => deleted);
+            }
+            return deleted;
+        };
+        if (options?.deleteBlocks) {
+            const store = this.properties.store;
+            const hashes = deleted.map((node) => node.hash);
+            const deleteResult = hasRmMany(store) && store.rmMany
+                ? store.rmMany(hashes)
+                : Promise.all(hashes.map((hash) => store.rm(hash))).then(() => { });
+            return mapMaybePromise(deleteResult, afterStoreDelete);
+        }
+        return afterStoreDelete();
+    }
+    finishDeleteMany(deletedByHash, nodes, storeHashes, options) {
+        const deleted = nodes
+            .map((node) => deletedByHash.get(node.hash))
+            .filter((node) => !!node);
+        if (deleted.length === 0) {
+            return [];
+        }
+        const store = this.properties.store;
+        const afterStoreDelete = () => {
+            this._length -= deleted.length;
+            const graph = this.properties.nativeGraph?.graph;
+            if (graph?.deleteMany) {
+                graph.deleteMany(storeHashes);
+            }
+            else {
+                for (const hash of storeHashes) {
+                    graph?.delete(hash);
+                }
+            }
+            if (!options?.skipNextHeadUpdates) {
+                const deletedNexts = [];
+                for (const node of deleted) {
+                    if (node.meta.type !== EntryType.CUT) {
+                        deletedNexts.push(...node.meta.next);
+                    }
+                }
+                return mapMaybePromise(this.privateUpdateNextHeadHashes(deletedNexts, true), () => deleted);
+            }
+            return deleted;
+        };
+        if (hasRmMany(store) && store.rmMany) {
+            return mapMaybePromise(store.rmMany(storeHashes), afterStoreDelete);
+        }
+        return Promise.all(storeHashes.map((hash) => store.rm(hash))).then(afterStoreDelete);
+    }
+    nativeLogEntryToShallowEntry(entry) {
+        return new ShallowEntry({
+            hash: entry.hash,
+            head: entry.head ?? false,
+            payloadSize: entry.payloadSize ?? 0,
+            meta: new ShallowMeta({
+                gid: entry.gid,
+                next: entry.next,
+                type: entry.type,
+                data: entry.data,
+                clock: new Clock({
+                    id: this.properties.publicKey.bytes,
+                    timestamp: new Timestamp({
+                        wallTime: BigInt(entry.clock.timestamp.wallTime),
+                        logical: entry.clock.timestamp.logical ?? 0,
+                    }),
+                }),
+            }),
+        });
+    }
+    nativeTrimmedHashToShallowEntry(hash) {
+        return new ShallowEntry({
+            hash,
+            head: false,
+            payloadSize: 0,
+            meta: new ShallowMeta({
+                gid: "",
+                next: [],
+                type: EntryType.APPEND,
+                clock: new Clock({
+                    id: this.properties.publicKey.bytes,
+                    timestamp: new Timestamp({
+                        wallTime: 0n,
+                        logical: 0,
+                    }),
+                }),
+            }),
+        });
+    }
+    nativeLogEntriesToShallowEntries(entries) {
+        return entries.map((entry) => this.nativeLogEntryToShallowEntry(entry));
+    }
     async getMemoryUsage() {
+        if (this.properties.nativeGraph) {
+            return this.properties.nativeGraph.graph.payloadSizeSum();
+        }
         const indexed = (await this.properties.index.sum({ key: "payloadSize" })) || 0;
-        const pending = [...this.pendingIndexWrites.values()].reduce((sum, entry) => sum + (entry.payloadSize || 0), 0);
-        return typeof indexed === "bigint" ? indexed + BigInt(pending) : indexed + pending;
+        let pending = 0;
+        for (const [hash, write] of this.pendingIndexWrites) {
+            pending +=
+                this.materializePendingIndexWrite(hash, write).payloadSize || 0;
+        }
+        return typeof indexed === "bigint"
+            ? indexed + BigInt(pending)
+            : indexed + pending;
     }
     async privateUpdateNextHeadProperty(from, isHead) {
         if (from.meta.type === EntryType.CUT) {
             // if the next is a cut, we can't update it, since it's not in the index
             return;
         }
-        for (const next of from.meta.next) {
-            const pending = this.pendingIndexWrites.get(next);
+        await this.privateUpdateNextHeadHashes(from.meta.next, isHead);
+    }
+    async privateUpdateNextHeadHashes(nexts, isHead) {
+        const hashes = [...new Set(nexts.filter(Boolean))];
+        if (hashes.length === 0) {
+            return;
+        }
+        const existingNexts = isHead && this.properties.nativeGraph
+            ? this.properties.nativeGraph.graph.hasMany(hashes)
+            : undefined;
+        for (const next of hashes) {
+            const pending = this.getPendingIndexWrite(next);
+            if (!pending && existingNexts && !existingNexts.has(next)) {
+                continue;
+            }
             const indexedEntry = pending
                 ? { id: toId(next), value: pending }
                 : await this.properties.index.get(toId(next));
@@ -465,6 +1820,17 @@ export class EntryIndex {
             }
         }
     }
+    async notifyShadowedGids(entry) {
+        if (!this.properties.onGidRemoved || entry.meta.next.length === 0) {
+            return;
+        }
+        const shadowedGids = this.properties.nativeGraph
+            ? new Set(this.properties.nativeGraph.graph.shadowedGids(entry.meta.gid, entry.meta.next, entry.hash))
+            : await this.findShadowedGids(entry);
+        if (shadowedGids.size > 0) {
+            this.properties.onGidRemoved([...shadowedGids]);
+        }
+    }
     async clear() {
         this.clearPendingIndexFlushTimer();
         const hashes = new Set(this.pendingIndexWrites.keys());
@@ -571,7 +1937,50 @@ export class EntryIndex {
         }
         return mem ? mem : undefined;
         /* }
-        return undefined; */
+            return undefined; */
+    }
+    async resolveMany(hashes, options) {
+        if (hashes.length === 0) {
+            return [];
+        }
+        if (!canBatchResolveFromStore(this.properties.store, options)) {
+            return Promise.all(hashes.map((hash) => this.resolve(hash, options)));
+        }
+        const coercedOptions = typeof options === "object" ? options : undefined;
+        const resolved = new Array(hashes.length);
+        const missingHashes = [];
+        const missingPositions = [];
+        for (let i = 0; i < hashes.length; i++) {
+            const hash = hashes[i];
+            const mem = this.cache.get(hash);
+            if (mem !== undefined) {
+                resolved[i] = mem ? mem : undefined;
+                continue;
+            }
+            missingHashes.push(hash);
+            missingPositions.push(i);
+        }
+        if (missingHashes.length === 0) {
+            return resolved;
+        }
+        const values = await this.properties.store.getMany(missingHashes, withDefaultRemoteReadPriority(coercedOptions));
+        for (let i = 0; i < values.length; i++) {
+            const hash = missingHashes[i];
+            const value = values[i];
+            if (!value) {
+                if (coercedOptions?.ignoreMissing !== true) {
+                    throw new Error("Failed to load entry from head with hash: " + hash);
+                }
+                continue;
+            }
+            const entry = deserialize(value, Entry);
+            this.properties.init(entry);
+            entry.hash = hash;
+            entry.size = value.length;
+            this.cache.add(hash, entry);
+            resolved[missingPositions[i]] = entry;
+        }
+        return resolved;
     }
     async resolveFromStore(k, options) {
         let coercedOptions = options;
@@ -614,6 +2023,19 @@ export class EntryIndex {
         return null;
     }
 }
+const createHashMatchQuery = (hashes) => hashes.length === 1
+    ? new StringMatch({
+        key: "hash",
+        value: hashes[0],
+        caseInsensitive: false,
+        method: StringMatchMethod.exact,
+    })
+    : new Or(hashes.map((hash) => new StringMatch({
+        key: "hash",
+        value: hash,
+        caseInsensitive: false,
+        method: StringMatchMethod.exact,
+    })));
 const toNativeLogEntry = (entry) => ({
     hash: entry.hash,
     gid: entry.meta.gid,
@@ -621,6 +2043,7 @@ const toNativeLogEntry = (entry) => ({
     type: entry.meta.type,
     head: entry.head,
     payloadSize: entry.payloadSize,
+    data: entry.meta.data,
     clock: {
         timestamp: {
             wallTime: entry.meta.clock.timestamp.wallTime,
@@ -628,6 +2051,28 @@ const toNativeLogEntry = (entry) => ({
         },
     },
 });
+const getResolveShape = (options) => options && options !== true && options.type === "shape"
+    ? options.shape
+    : undefined;
+const isHeadHashOnlyShape = (shape) => {
+    const keys = Object.keys(shape);
+    return keys.length === 1 && shape.hash === true;
+};
+const isHeadDataShape = (shape) => {
+    const keys = Object.keys(shape);
+    if (keys.some((key) => key !== "hash" && key !== "meta")) {
+        return false;
+    }
+    if (shape.hash !== undefined && shape.hash !== true) {
+        return false;
+    }
+    const meta = shape.meta;
+    if (!meta || typeof meta !== "object") {
+        return false;
+    }
+    const metaKeys = Object.keys(meta);
+    return metaKeys.length === 1 && meta.data === true;
+};
 const projectShape = (value, shape) => {
     const out = {};
     for (const [key, selector] of Object.entries(shape)) {