@peerbit/log 3.0.34-aa577a5 → 3.0.34-cccc078

package/dist/src/log.js

@@ -8,50 +8,53 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 var Log_1;
-import { SignatureWithKey, randomBytes, sha256Base64Sync, X25519Keypair } from "@peerbit/crypto";
-import { Cache } from "@peerbit/cache";
+import { deserialize, field, fixedArray, variant } from "@dao-xyz/borsh";
 import {} from "@peerbit/any-store";
-import { EntryIndex } from "./entry-index.js";
+import { cidifyString } from "@peerbit/blocks-interface";
+import { SignatureWithKey, X25519Keypair, randomBytes, sha256Base64Sync, } from "@peerbit/crypto";
+import {} from "@peerbit/indexer-interface";
+import { create } from "@peerbit/indexer-sqlite3";
+import {} from "@peerbit/keychain";
+import {} from "./change.js";
+import { LamportClock as Clock, HLC, LamportClock, Timestamp, } from "./clock.js";
+import { NO_ENCODING } from "./encoding.js";
+import { EntryIndex, } from "./entry-index.js";
+import {} from "./entry-with-refs.js";
+import { Entry, EntryType, Payload, ShallowEntry, } from "./entry.js";
+import { findUniques } from "./find-uniques.js";
 import * as LogError from "./log-errors.js";
 import * as Sorting from "./log-sorting.js";
-import { findUniques } from "./find-uniques.js";
-import { Entry, Payload, EntryType } from "./entry.js";
-import { HLC, LamportClock as Clock, LamportClock, Timestamp } from "./clock.js";
-import { deserialize, field, fixedArray, variant } from "@dao-xyz/borsh";
-import { NO_ENCODING } from "./encoding.js";
-import { HeadsIndex } from "./heads.js";
-import { Values } from "./values.js";
 import { Trim } from "./trim.js";
-import { logger } from "./logger.js";
-import {} from "./change.js";
-import {} from "./entry-with-refs.js";
-import {} from "@peerbit/blocks-interface";
-import { cidifyString } from "@peerbit/blocks";
-import {} from "@peerbit/keychain";
-const { LastWriteWins, NoZeroes } = Sorting;
-const ENTRY_CACHE_MAX = 1000; // TODO as param
+const { LastWriteWins } = Sorting;
+export const ENTRY_JOIN_SHAPE = {
+    hash: true,
+    meta: { type: true, next: true, gid: true, clock: true },
+};
 let Log = Log_1 = class Log {
     _id;
-    _sortFn;
+    /* private _sortFn!: Sorting.ISortFunction; */
     _storage;
     _hlc;
     // Identity
     _identity;
     // Keeping track of entries
     _entryIndex;
-    _headsIndex;
-    _values;
+    /* 	private _headsIndex!: HeadsIndex<T>;
+        private _values!: Values<T>;
+     */
     // Index of all next pointers in this log
-    _nextsIndex;
+    /* private _nextsIndex!: Map<string, Set<string>>; */
     _keychain;
     _encoding;
     _trim;
-    _entryCache;
     _canAppend;
     _onChange;
     _closed = true;
-    _memory;
+    _closeController;
+    _loadedOnce = false;
+    _indexer;
     _joining; // entry hashes that are currently joining into this log
+    _sortFn;
     constructor(properties) {
         this._id = properties?.id || randomBytes(32);
     }
@@ -65,18 +68,13 @@ let Log = Log_1 = class Log {
         if (this.closed === false) {
             throw new Error("Already open");
         }
-        const { encoding, trim, keychain, cache, onGidRemoved } = options;
-        let { sortFn } = options;
-        if (sortFn == null) {
-            sortFn = LastWriteWins;
-        }
-        sortFn = sortFn;
-        this._sortFn = NoZeroes(sortFn);
+        this._closeController = new AbortController();
+        const { encoding, trim, keychain, indexer, onGidRemoved, sortFn } = options;
+        // TODO do correctly with tie breaks
+        this._sortFn = sortFn || LastWriteWins;
         this._storage = store;
-        this._memory = cache;
-        if (this._memory && (await this._memory.status()) !== "open") {
-            await this._memory.open();
-        }
+        this._indexer = indexer || (await create());
+        await this._indexer.start?.();
         this._encoding = encoding || NO_ENCODING;
         this._joining = new Map();
         // Identity
@@ -85,47 +83,30 @@ let Log = Log_1 = class Log {
         this._keychain = keychain;
         // Clock
         this._hlc = new HLC();
-        this._nextsIndex = new Map();
         const id = this.id;
         if (!id) {
             throw new Error("Id not set");
         }
-        this._headsIndex = new HeadsIndex(id);
-        await this._headsIndex.init(this, { onGidRemoved });
-        this._entryCache = new Cache({ max: ENTRY_CACHE_MAX });
         this._entryIndex = new EntryIndex({
             store: this._storage,
             init: (e) => e.init(this),
-            cache: this._entryCache
+            onGidRemoved,
+            index: await (await this._indexer.scope("heads")).init({ schema: ShallowEntry }),
+            publicKey: this._identity.publicKey,
+            sort: this._sortFn,
         });
-        this._values = new Values(this._entryIndex, this._sortFn);
+        await this._entryIndex.init();
+        /* 	this._values = new Values(this._entryIndex, this._sortFn); */
         this._trim = new Trim({
-            headsIndex: this._headsIndex,
+            index: this._entryIndex,
             deleteNode: async (node) => {
-                // TODO check if we have before delete?
-                const entry = await this.get(node.value);
-                //f (!!entry)
-                const a = this.values.length;
-                if (entry) {
-                    this.values.deleteNode(node);
-                    await Promise.all([
-                        this.headsIndex.del(this.entryIndex.getShallow(node.value)),
-                        this.entryIndex.delete(node.value)
-                    ]);
-                    this.nextsIndex.delete(node.value);
-                    await this.blocks.rm(node.value);
-                }
-                const b = this.values.length;
-                if (a === b) {
-                    /* throw new Error(
-                        "Unexpected miss match between log size and entry index size: " +
-                        this.values.length +
-                        this.entryIndex._index.size
-                    ); */
-                }
-                return entry;
+                const resolved = await this.get(node.hash);
+                await this._entryIndex.delete(node.hash);
+                await this._storage.rm(node.hash);
+                return resolved;
             },
-            values: () => this.values
+            sortFn: this._sortFn,
+            getLength: () => this.length,
         }, trim);
         this._canAppend = async (entry) => {
             if (options?.canAppend) {
@@ -137,6 +118,7 @@ let Log = Log_1 = class Log {
         };
         this._onChange = options?.onChange;
         this._closed = false;
+        this._closeController = new AbortController();
     }
     _idString;
     get idString() {
@@ -162,13 +144,10 @@ let Log = Log_1 = class Log {
      * Returns the length of the log.
      */
     get length() {
-        return this.values.length;
-    }
-    get values() {
-        if (this.closed) {
+        if (this._closed) {
             throw new Error("Closed");
         }
-        return this._values;
+        return this._entryIndex.length;
     }
     get canAppend() {
         return this._canAppend;
@@ -179,36 +158,20 @@ let Log = Log_1 = class Log {
      * @returns {boolean}
      */
     has(cid) {
-        return this._entryIndex._index.has(cid);
+        return this._entryIndex.has(cid);
     }
     /**
      * Get all entries sorted. Don't use this method anywhere where performance matters
      */
-    toArray() {
+    async toArray() {
         // we call init, because the values might be unitialized
-        return this._values.toArray().then((arr) => arr.map((x) => x.init(this)));
+        return this.entryIndex.query([], this.sortFn.sort, true).all();
     }
     /**
      * Returns the head index
      */
-    get headsIndex() {
-        return this._headsIndex;
-    }
-    get memory() {
-        return this._memory;
-    }
-    async getHeads() {
-        const heads = new Array(this.headsIndex.index.size);
-        let i = 0;
-        for (const hash of this.headsIndex.index) {
-            heads[i++] = this._entryIndex.get(hash).then((x) => x?.init(this));
-        }
-        const resolved = await Promise.all(heads);
-        const defined = resolved.filter((x) => !!x);
-        if (defined.length !== resolved.length) {
-            logger.error("Failed to resolve all heads");
-        }
-        return defined;
+    getHeads(resolve = false) {
+        return this.entryIndex.getHeads(undefined, resolve);
     }
     /**
      * Returns an array of Entry objects that reference entries which
@@ -238,9 +201,6 @@ let Log = Log_1 = class Log {
     get blocks() {
         return this._storage;
     }
-    get nextsIndex() {
-        return this._nextsIndex;
-    }
     get entryIndex() {
         return this._entryIndex;
     }
@@ -264,68 +224,89 @@ let Log = Log_1 = class Log {
         this._identity = identity;
     }
     /**
-     * Find an entry.
+     * Get an entry.
      * @param {string} [hash] The hashes of the entry
      */
     get(hash, options) {
-        return this._entryIndex.get(hash, options);
+        return this._entryIndex.get(hash, options ? { type: "full", timeout: options.timeout } : undefined);
     }
-    async traverse(rootEntries, amount = -1, endHash) {
-        // Sort the given given root entries and use as the starting stack
-        let stack = rootEntries.sort(this._sortFn).reverse();
-        // Cache for checking if we've processed an entry already
-        let traversed = {};
-        // End result
-        const result = {};
-        let count = 0;
-        // Named function for getting an entry from the log
-        const getEntry = (e) => this.get(e);
-        // Add an entry to the stack and traversed nodes index
-        const addToStack = (entry) => {
-            // If we've already processed the Entry<T>, don't add it to the stack
-            if (!entry || traversed[entry.hash]) {
-                return;
-            }
-            // Add the entry in front of the stack and sort
-            stack = [entry, ...stack].sort(this._sortFn).reverse();
-            // Add to the cache of processed entries
-            traversed[entry.hash] = true;
-        };
-        const addEntry = (rootEntry) => {
-            result[rootEntry.hash] = rootEntry;
-            traversed[rootEntry.hash] = true;
-            count++;
-        };
-        // Start traversal
-        // Process stack until it's empty (traversed the full log)
-        // or when we have the requested amount of entries
-        // If requested entry amount is -1, traverse all
-        while (stack.length > 0 && (count < amount || amount < 0)) {
-            // eslint-disable-line no-unmodified-loop-condition
-            // Get the next element from the stack
-            const entry = stack.shift();
-            if (!entry) {
-                throw new Error("Unexpected");
-            }
-            // Add to the result
-            addEntry(entry);
-            // If it is the specified end hash, break out of the while loop
-            if (endHash && endHash === entry.hash)
-                break;
-            // Add entry's next references to the stack
-            const entries = (await Promise.all(entry.next.map(getEntry))).filter((x) => !!x);
-            entries.forEach(addToStack);
+    /**
+     * Get a entry with shallow representation
+     * @param {string} [hash] The hashes of the entry
+     */
+    async getShallow(hash, options) {
+        return (await this._entryIndex.getShallow(hash))?.value;
+    }
+    /*
+        async traverse(
+            rootEntries: Entry<T>[],
+            amount = -1,
+            endHash?: string
+        ): Promise<{ [key: string]: Entry<T> }> {
+            // Sort the given given root entries and use as the starting stack
+            let stack: Entry<T>[] = rootEntries.sort(this._sortFn).reverse();
+    
+            // Cache for checking if we've processed an entry already
+            let traversed: { [key: string]: boolean } = {};
+            // End result
+            const result: { [key: string]: Entry<T> } = {};
+            let count = 0;
+            // Named function for getting an entry from the log
+            const getEntry = (e: string) => this.get(e);
+    
+            // Add an entry to the stack and traversed nodes index
+            const addToStack = (entry: Entry<T>) => {
+                // If we've already processed the Entry<T>, don't add it to the stack
+                if (!entry || traversed[entry.hash]) {
+                    return;
+                }
+    
+                // Add the entry in front of the stack and sort
+                stack = [entry, ...stack].sort(this._sortFn).reverse();
+                // Add to the cache of processed entries
+                traversed[entry.hash] = true;
+            };
+    
+            const addEntry = (rootEntry: Entry<T>) => {
+                result[rootEntry.hash] = rootEntry;
+                traversed[rootEntry.hash] = true;
+                count++;
+            };
+    
+            // Start traversal
+            // Process stack until it's empty (traversed the full log)
+            // or when we have the requested amount of entries
+            // If requested entry amount is -1, traverse all
+            while (stack.length > 0 && (count < amount || amount < 0)) {
+                // eslint-disable-line no-unmodified-loop-condition
+                // Get the next element from the stack
+                const entry = stack.shift();
+                if (!entry) {
+                    throw new Error("Unexpected");
+                }
+                // Add to the result
+                addEntry(entry);
+                // If it is the specified end hash, break out of the while loop
+                if (endHash && endHash === entry.hash) break;
+    
+                // Add entry's next references to the stack
+                const entries = (await Promise.all(entry.next.map(getEntry))).filter(
+                    (x) => !!x
+                ) as Entry<any>[];
+                entries.forEach(addToStack);
+            }
+    
+            stack = [];
+            traversed = {};
+            // End result
+            return result;
         }
-        stack = [];
-        traversed = {};
-        // End result
-        return result;
-    }
+     */
     async getReferenceSamples(from, options) {
         const hashes = new Set();
         const pointerCount = options?.pointerCount || 0;
         const memoryLimit = options?.memoryLimit;
-        const maxDistance = Math.min(pointerCount, this._values.length);
+        const maxDistance = Math.min(pointerCount, this.entryIndex.length);
         if (maxDistance === 0) {
             return [];
         }
@@ -383,8 +364,9 @@ let Log = Log_1 = class Log {
     }
     /**
      * Append an entry to the log.
-     * @param {Entry} entry Entry to add
-     * @return {Log} New Log containing the appended value
+     * @param {T} data The data to be appended
+     * @param {AppendOptions} [options] The options for the append
+     * @returns {{ entry: Entry<T>; removed: ShallowEntry[] }} The appended entry and an array of removed entries
      */
     async append(data, options = {}) {
         // Update the clock (find the latest clock)
@@ -395,11 +377,14 @@ let Log = Log_1 = class Log {
             }
         }
         await this.load({ reload: false });
-        const nexts = options.meta?.next || (await this.getHeads());
+        const nexts = options.meta?.next ||
+            (await this.entryIndex
+                .getHeads(undefined, { type: "shape", shape: Sorting.ENTRY_SORT_SHAPE })
+                .all());
         // Calculate max time for log/graph
         const clock = new Clock({
             id: this._identity.publicKey.bytes,
-            timestamp: options?.meta?.timestamp || this._hlc.now()
+            timestamp: options?.meta?.timestamp || this._hlc.now(),
         });
         const entry = await Entry.create({
             store: this._storage,
@@ -411,39 +396,47 @@ let Log = Log_1 = class Log {
                 type: options.meta?.type,
                 gidSeed: options.meta?.gidSeed,
                 data: options.meta?.data,
-                next: nexts
+                next: nexts,
             },
             encoding: this._encoding,
             encryption: options.encryption
                 ? {
                     keypair: options.encryption.keypair,
                     receiver: {
-                        ...options.encryption.receiver
-                    }
+                        ...options.encryption.receiver,
+                    },
                 }
                 : undefined,
-            canAppend: options.canAppend || this._canAppend
+            canAppend: options.canAppend || this._canAppend,
         });
         if (!entry.hash) {
             throw new Error("Unexpected");
         }
-        for (const e of nexts) {
-            if (!this.has(e.hash)) {
-                await this.join([e]);
-            }
-            let nextIndexSet = this._nextsIndex.get(e.hash);
-            if (!nextIndexSet) {
-                nextIndexSet = new Set();
-                nextIndexSet.add(entry.hash);
-                this._nextsIndex.set(e.hash, nextIndexSet);
-            }
-            else {
-                nextIndexSet.add(entry.hash);
+        if (entry.meta.type !== EntryType.CUT) {
+            for (const e of nexts) {
+                if (!(await this.has(e.hash))) {
+                    let entry;
+                    if (e instanceof Entry) {
+                        entry = e;
+                    }
+                    else {
+                        let resolved = await this.entryIndex.get(e.hash);
+                        if (!resolved) {
+                            // eslint-disable-next-line no-console
+                            console.warn("Unexpected missing entry when joining", e.hash);
+                            continue;
+                        }
+                        entry = resolved;
+                    }
+                    await this.join([entry]);
+                }
             }
         }
-        await this._entryIndex.set(entry, false); // save === false, because its already saved when Entry.create
-        await this._headsIndex.put(entry, { cache: { update: false } }); // we will update the cache a few lines later *
-        await this._values.put(entry);
+        await this.entryIndex.put(entry, {
+            unique: true,
+            isHead: true,
+            toMultiHash: false,
+        });
         const removed = await this.processEntry(entry);
         entry.init({ encoding: this._encoding, keychain: this._keychain });
         const trimmed = await this.trim(options?.trim);
@@ -454,22 +447,16 @@ let Log = Log_1 = class Log {
         }
         const changes = {
             added: [entry],
-            removed: removed
+            removed,
         };
-        await this._headsIndex.updateHeadsCache(changes); // * here
         await (options?.onChange || this._onChange)?.(changes);
         return { entry, removed };
     }
     async reset(entries) {
-        this._nextsIndex = new Map();
-        this._entryIndex = new EntryIndex({
-            store: this._storage,
-            init: (e) => e.init(this),
-            cache: this._entryCache
-        });
-        await this.headsIndex.reset([]);
-        this._values = new Values(this._entryIndex, this._sortFn, []);
-        await this.join(entries);
+        const heads = await this.getHeads(true).all();
+        await this._entryIndex.clear();
+        await this._onChange?.({ added: [], removed: heads });
+        await this.join(entries || heads);
     }
     async remove(entry, options) {
         await this.load({ reload: false });
@@ -477,269 +464,200 @@ let Log = Log_1 = class Log {
         if (entries.length === 0) {
             return {
                 added: [],
-                removed: []
+                removed: [],
             };
         }
+        const change = {
+            added: [],
+            removed: Array.isArray(entry) ? entry : [entry],
+        };
+        await this._onChange?.(change);
         if (options?.recursively) {
             await this.deleteRecursively(entry);
         }
         else {
             for (const entry of entries) {
-                await this.delete(entry);
+                await this.delete(entry.hash);
             }
         }
-        const change = {
-            added: [],
-            removed: Array.isArray(entry) ? entry : [entry]
-        };
-        /* 	await Promise.all([
-                this._logCache?.queue(change),
-                this._onUpdate(change),
-            ]); */
-        await this._onChange?.(change);
         return change;
     }
-    iterator(options) {
+    /* iterator(options?: {
+        from?: "tail" | "head";
+        amount?: number;
+    }): IterableIterator<string> {
         const from = options?.from || "tail";
         const amount = typeof options?.amount === "number" ? options?.amount : -1;
         let next = from === "tail" ? this._values.tail : this._values.head;
-        const nextFn = from === "tail" ? (e) => e.prev : (e) => e.next;
+        const nextFn = from === "tail" ? (e: any) => e.prev : (e: any) => e.next;
         return (function* () {
             let counter = 0;
             while (next) {
                 if (amount >= 0 && counter >= amount) {
                     return;
                 }
+
                 yield next.value;
                 counter++;
+
                 next = nextFn(next);
             }
         })();
-    }
+    } */
     async trim(option = this._trim.options) {
         return this._trim.trim(option);
     }
-    /**
-     *
-     * @param entries
-     * @returns change
-     */
-    /* async sync(
-        entries: (EntryWithRefs<T> | Entry<T> | string)[],
-        options: {
-            canAppend?: CanAppend<T>;
-            onChange?: (change: Change<T>) => void | Promise<void>;
-            timeout?: number;
-        } = {}
-    ): Promise<void> {
-
-
-        logger.debug(`Sync request #${entries.length}`);
-        const entriesToJoin: (Entry<T> | string)[] = [];
-        for (const e of entries) {
-            if (e instanceof Entry || typeof e === "string") {
-                entriesToJoin.push(e);
-            } else {
-                for (const ref of e.references) {
-                    entriesToJoin.push(ref);
-                }
-                entriesToJoin.push(e.entry);
-            }
-        }
-
-        await this.join(entriesToJoin, {
-            canAppend: (entry) => {
-                const canAppend = options?.canAppend || this.canAppend;
-                return !canAppend || canAppend(entry);
-            },
-            onChange: (change) => {
-                options?.onChange?.(change);
-                return this._onChange?.({
-                    added: change.added,
-                    removed: change.removed,
-                });
-            },
-            timeout: options.timeout,
-        });
-    } */
     async join(entriesOrLog, options) {
-        const definedOptions = {
-            ...options,
-            cache: options?.cache ?? {
-                update: true
+        let entries;
+        let references = new Map();
+        if (entriesOrLog instanceof Log_1) {
+            if (entriesOrLog.entryIndex.length === 0)
+                return;
+            entries = await entriesOrLog.toArray();
+            for (const element of entries) {
+                references.set(element.hash, element);
             }
-        };
-        await this.load({ reload: false });
-        if (entriesOrLog.length === 0) {
-            return;
         }
-        /* const joinLength = options?.length ?? Number.MAX_SAFE_INTEGER;  TODO */
-        const visited = new Set();
-        const nextRefs = new Map();
-        const entriesBottomUp = [];
-        const stack = [];
-        const resolvedEntries = new Map();
-        const entries = Array.isArray(entriesOrLog)
-            ? entriesOrLog
-            : await entriesOrLog.values.toArray();
-        // Build a list of already resolved entries, and filter out already joined entries
-        for (const e of entries) {
-            // TODO, do this less ugly
-            let hash;
-            if (e instanceof Entry) {
-                hash = e.hash;
-                resolvedEntries.set(e.hash, e);
-                if (this.has(hash)) {
-                    continue;
-                }
-                stack.push(hash);
+        else if (Array.isArray(entriesOrLog)) {
+            if (entriesOrLog.length === 0) {
+                return;
             }
-            else if (typeof e === "string") {
-                hash = e;
-                if (this.has(hash)) {
-                    continue;
+            entries = [];
+            for (const element of entriesOrLog) {
+                if (element instanceof Entry) {
+                    entries.push(element);
+                    references.set(element.hash, element);
                 }
-                stack.push(hash);
-            }
-            else {
-                hash = e.entry.hash;
-                resolvedEntries.set(e.entry.hash, e.entry);
-                if (this.has(hash)) {
-                    continue;
+                else if (typeof element === "string") {
+                    let entry = await Entry.fromMultihash(this._storage, element, {
+                        timeout: options?.timeout,
+                    });
+                    if (!entry) {
+                        throw new Error("Missing entry in join by hash: " + element);
+                    }
+                    entries.push(entry);
+                }
+                else if (element instanceof ShallowEntry) {
+                    let entry = await Entry.fromMultihash(this._storage, element.hash, {
+                        timeout: options?.timeout,
+                    });
+                    if (!entry) {
+                        throw new Error("Missing entry in join by hash: " + element.hash);
+                    }
+                    entries.push(entry);
                 }
-                stack.push(hash);
-                for (const e2 of e.references) {
-                    resolvedEntries.set(e2.hash, e2);
-                    if (this.has(e2.hash)) {
-                        continue;
+                else {
+                    entries.push(element.entry);
+                    references.set(element.entry.hash, element.entry);
+                    for (const ref of element.references) {
+                        references.set(ref.hash, ref);
                     }
-                    stack.push(e2.hash);
                 }
             }
         }
-        const removedHeads = [];
-        for (const hash of stack) {
-            if (visited.has(hash) || this.has(hash)) {
-                continue;
+        else {
+            let all = await entriesOrLog.all(); // TODO dont load all at once
+            if (all.length === 0) {
+                return;
             }
-            visited.add(hash);
-            const entry = resolvedEntries.get(hash) ||
-                (await Entry.fromMultihash(this._storage, hash, {
-                    timeout: options?.timeout
-                }));
-            entry.init(this);
-            resolvedEntries.set(entry.hash, entry);
-            let nexts;
-            if (entry.meta.type !== EntryType.CUT &&
-                (nexts = await entry.getNext())) {
-                let isRoot = true;
-                for (const next of nexts) {
-                    if (!this.has(next)) {
-                        isRoot = false;
-                    }
-                    else {
-                        if (this._headsIndex.has(next)) {
-                            const toRemove = (await this.get(next, options));
-                            await this._headsIndex.del(toRemove);
-                            removedHeads.push(toRemove);
-                        }
-                    }
-                    let nextIndexSet = nextRefs.get(next);
-                    if (!nextIndexSet) {
-                        nextIndexSet = [];
-                        nextIndexSet.push(entry);
-                        nextRefs.set(next, nextIndexSet);
-                    }
-                    else {
-                        nextIndexSet.push(entry);
-                    }
-                    if (!visited.has(next)) {
-                        stack.push(next);
-                    }
-                }
-                if (isRoot) {
-                    entriesBottomUp.push(entry);
-                }
+            entries = all;
+        }
+        let heads = new Map();
+        for (const entry of entries) {
+            if (heads.has(entry.hash)) {
+                continue;
             }
-            else {
-                entriesBottomUp.push(entry);
+            heads.set(entry.hash, true);
+            for (const next of await entry.getNext()) {
+                heads.set(next, false);
             }
         }
-        while (entriesBottomUp.length > 0) {
-            const e = entriesBottomUp.shift();
-            await this._joining.get(e.hash);
-            const p = this.joinEntry(e, nextRefs, entriesBottomUp, definedOptions).then(() => this._joining.delete(e.hash) // TODO, if head we run into problems with concurrency here!, we add heads at line 929 but resolve here
-            );
-            this._joining.set(e.hash, p);
+        for (const entry of entries) {
+            const p = this.joinRecursively(entry, {
+                references,
+                isHead: heads.get(entry.hash),
+                ...options,
+            });
+            this._joining.set(entry.hash, p);
+            p.finally(() => {
+                this._joining.delete(entry.hash);
+            });
             await p;
         }
     }
-    async joinEntry(e, nextRefs, stack, options) {
-        if (this.length > (options?.length ?? Number.MAX_SAFE_INTEGER)) {
+    /**
+     * Bottom up join of entries into the log
+     * @param entry
+     * @param options
+     * @returns
+     */
+    async joinRecursively(entry, options) {
+        if (this.entryIndex.length > (options?.length ?? Number.MAX_SAFE_INTEGER)) {
             return;
         }
-        if (!e.hash) {
+        if (!entry.hash) {
             throw new Error("Unexpected");
         }
-        const headsWithGid = this.headsIndex.gids.get(e.gid);
+        if (await this.has(entry.hash)) {
+            return;
+        }
+        entry.init(this);
+        if (options?.verifySignatures) {
+            if (!(await entry.verifySignatures())) {
+                throw new Error('Invalid signature entry with hash "' + entry.hash + '"');
+            }
+        }
+        if (this?._canAppend && !(await this?._canAppend(entry))) {
+            return;
+        }
+        const headsWithGid = await this.entryIndex
+            .getHeads(entry.gid, { type: "shape", shape: ENTRY_JOIN_SHAPE })
+            .all();
         if (headsWithGid) {
-            for (const [_k, v] of headsWithGid) {
-                if (v.meta.type === EntryType.CUT && v.next.includes(e.hash)) {
+            for (const v of headsWithGid) {
+                // TODO second argument should be a time compare instead? what about next nexts?
+                // and check the cut entry is newer than the current 'entry'
+                if (v.meta.type === EntryType.CUT &&
+                    v.meta.next.includes(entry.hash) &&
+                    Sorting.compare(entry, v, this._sortFn) < 0) {
                     return; // already deleted
                 }
             }
         }
-        if (!this.has(e.hash)) {
-            if (options?.verifySignatures) {
-                if (!(await e.verifySignatures())) {
-                    throw new Error('Invalid signature entry with hash "' + e.hash + '"');
-                }
-            }
-            if (this?._canAppend && !(await this?._canAppend(e))) {
-                return;
-            }
-            // Update the internal entry index
-            await this._entryIndex.set(e);
-            await this._values.put(e);
-            if (e.meta.type !== EntryType.CUT) {
-                for (const a of e.next) {
-                    if (!this.has(a)) {
-                        await this.join([a]);
-                    }
-                    let nextIndexSet = this._nextsIndex.get(a);
-                    if (!nextIndexSet) {
-                        nextIndexSet = new Set();
-                        nextIndexSet.add(e.hash);
-                        this._nextsIndex.set(a, nextIndexSet);
-                    }
-                    else {
-                        nextIndexSet.add(a);
+        if (entry.meta.type !== EntryType.CUT) {
+            for (const a of entry.next) {
+                if (!(await this.has(a))) {
+                    const nested = options.references?.get(a) ||
+                        (await Entry.fromMultihash(this._storage, a, {
+                            timeout: options?.timeout,
+                        }));
+                    if (!nested) {
+                        throw new Error("Missing entry in joinRecursively: " + a);
                     }
+                    const p = this.joinRecursively(nested, options.isHead ? { ...options, isHead: false } : options);
+                    this._joining.set(nested.hash, p);
+                    p.finally(() => {
+                        this._joining.delete(nested.hash);
+                    });
+                    await p;
                 }
             }
-            const clock = await e.getClock();
-            this._hlc.update(clock.timestamp);
-            const removed = await this.processEntry(e);
-            const trimmed = await this.trim(options?.trim);
-            if (trimmed) {
-                for (const entry of trimmed) {
-                    removed.push(entry);
-                }
-            }
-            await this?._onChange?.({ added: [e], removed: removed });
         }
-        const forward = nextRefs.get(e.hash);
-        if (forward) {
-            if (this._headsIndex.has(e.hash)) {
-                await this._headsIndex.del(e, options);
-            }
-            for (const en of forward) {
-                stack.push(en);
+        const clock = await entry.getClock();
+        this._hlc.update(clock.timestamp);
+        await this._entryIndex.put(entry, {
+            unique: false,
+            isHead: options.isHead,
+            toMultiHash: true,
+        });
+        const removed = await this.processEntry(entry);
+        const trimmed = await this.trim(options?.trim);
+        if (trimmed) {
+            for (const entry of trimmed) {
+                removed.push(entry);
             }
         }
-        else {
-            await this.headsIndex.put(e, options);
-        }
+        await this?._onChange?.({ added: [entry], removed: removed });
     }
     async processEntry(entry) {
         if (entry.meta.type === EntryType.CUT) {
@@ -755,22 +673,25 @@ let Log = Log_1 = class Log {
         const deleted = [];
         while (stack.length > 0) {
             const entry = stack.pop();
-            if ((counter > 0 || !skipFirst) && this.has(entry.hash)) {
-                // TODO test last argument: It is for when multiple heads point to the same entry, hence we might visit it multiple times? or a concurrent delete process is doing it before us.
-                this._trim.deleteFromCache(entry);
-                await this._headsIndex.del(entry);
-                await this._values.delete(entry);
-                await this._entryIndex.delete(entry.hash);
-                this._nextsIndex.delete(entry.hash);
-                deleted.push(entry);
-                promises.push(entry.delete(this._storage));
-            }
-            for (const next of entry.next) {
-                const nextFromNext = this._nextsIndex.get(next);
-                if (nextFromNext) {
-                    nextFromNext.delete(entry.hash);
+            const skip = counter === 0 && skipFirst;
+            if (!skip) {
+                const has = await this.has(entry.hash);
+                if (has) {
+                    // TODO test last argument: It is for when multiple heads point to the same entry, hence we might visit it multiple times? or a concurrent delete process is doing it before us.
+                    const deletedEntry = await this.delete(entry.hash);
+                    if (deletedEntry) {
+                        /* 	this._nextsIndex.delete(entry.hash); */
+                        deleted.push(deletedEntry);
+                    }
                 }
-                if (!nextFromNext || nextFromNext.size === 0) {
+            }
+            for (const next of entry.meta.next) {
+                const nextFromNext = this.entryIndex.getHasNext(next);
+                const entriesThatHasNext = await nextFromNext.all();
+                // if there are no entries which is not of "CUT" type, we can safely delete the next entry
+                // figureately speaking, these means where are cutting all branches to a stem, so we can delete the stem as well
+                let hasAlternativeNext = !!entriesThatHasNext.find((x) => x.meta.type !== EntryType.CUT);
+                if (!hasAlternativeNext) {
                     const ne = await this.get(next);
                     if (ne) {
                         stack.push(ne);
@@ -782,33 +703,10 @@ let Log = Log_1 = class Log {
         await Promise.all(promises);
         return deleted;
     }
-    async delete(entry) {
-        this._trim.deleteFromCache(entry);
-        await this._headsIndex.del(entry, {
-            cache: {
-                update: false
-            }
-        }); // cache is not updated here, but at *
-        await this._values.delete(entry);
-        await this._entryIndex.delete(entry.hash);
-        this._nextsIndex.delete(entry.hash);
-        const newHeads = [];
-        for (const next of entry.next) {
-            const ne = await this.get(next);
-            if (ne) {
-                const nexts = this._nextsIndex.get(next);
-                nexts.delete(entry.hash);
-                if (nexts.size === 0) {
-                    await this._headsIndex.put(ne);
-                    newHeads.push(ne.hash);
-                }
-            }
-        }
-        await this._headsIndex.updateHeadsCache({
-            added: newHeads,
-            removed: [entry.hash]
-        }); // * here
-        return entry.delete(this._storage);
+    async delete(hash) {
+        await this._trim.deleteFromCache(hash);
+        const removedEntry = await this._entryIndex.delete(hash);
+        return removedEntry;
     }
     /**
      * Returns the log entries as a formatted string.
@@ -828,37 +726,29 @@ let Log = Log_1 = class Log {
             let padding = new Array(Math.max(len - 1, 0));
             padding = len > 1 ? padding.fill("  ") : padding;
             padding = len > 0 ? padding.concat(["└─"]) : padding;
-            /* istanbul ignore next */
             return (padding.join("") +
-                (payloadMapper ? payloadMapper(e.payload) : e.payload));
+                (payloadMapper?.(e.payload) || e.payload));
         }))).join("\n");
     }
-    async idle() {
-        await this._headsIndex.headsCache?.idle();
-    }
     async close() {
         // Don't return early here if closed = true, because "load" might create processes that needs to be closed
         this._closed = true; // closed = true before doing below, else we might try to open the headsIndex cache because it is closed as we assume log is still open
-        await this._entryCache?.clear();
-        await this._headsIndex?.close();
-        await this._memory?.close();
-        this._entryCache = undefined;
-        this._headsIndex = undefined;
-        this._memory = undefined;
-        this._values = undefined;
+        this._closeController.abort();
+        await this._indexer?.stop?.();
+        this._indexer = undefined;
+        this._loadedOnce = false;
     }
     async drop() {
         // Don't return early here if closed = true, because "load" might create processes that needs to be closed
         this._closed = true; // closed = true before doing below, else we might try to open the headsIndex cache because it is closed as we assume log is still open
-        await Promise.all([...this.entryIndex._index.values()].map((x) => this.blocks.rm(x.hash)));
-        await this._headsIndex?.drop();
-        await this._entryCache?.clear();
-        await this._memory?.clear();
-        await this._memory?.close();
+        this._closeController.abort();
+        await this.entryIndex?.clear();
+        await this._indexer?.drop();
+        await this._indexer?.stop?.();
     }
     async recover() {
         // merge existing
-        const existing = await this.getHeads();
+        const existing = await this.getHeads(true).all();
         const allHeads = new Map();
         for (const head of existing) {
             allHeads.set(head.hash, head);
@@ -887,42 +777,33 @@ let Log = Log_1 = class Log {
         // assume they are valid, (let access control reject them if not)
         await this.load({ reload: true, heads: [...allHeads.values()] });
     }
-    async load(opts = { reload: true }) {
+    async load(opts = {}) {
         if (this.closed) {
             throw new Error("Closed");
         }
+        if (this._loadedOnce && !opts.reload && !opts.reset) {
+            return;
+        }
+        this._loadedOnce = true;
         const providedCustomHeads = Array.isArray(opts["heads"]);
         const heads = providedCustomHeads
             ? opts["heads"]
-            : await this.headsIndex.load({
-                replicate: true, // TODO this.replication.replicate(x) => true/false
-                timeout: opts.fetchEntryTimeout,
-                reload: opts.reload,
+            : await this._entryIndex
+                .getHeads(undefined, {
+                type: "full",
+                signal: this._closeController.signal,
                 ignoreMissing: opts.ignoreMissing,
-                cache: { update: true, reset: true }
-            });
+                timeout: opts.timeout,
+            })
+                .all();
         if (heads) {
             // Load the log
-            if (providedCustomHeads) {
+            if (providedCustomHeads || opts.reset) {
                 await this.reset(heads);
             }
             else {
-                /*
-                TODO feat amount load
-                const amount = (opts as { amount?: number }).amount;
-                if (amount != null && amount >= 0 && amount < heads.length) {
-                    throw new Error(
-                        "You are not loading all heads, this will lead to unexpected behaviours on write. Please load at least load: " +
-                        amount +
-                        " entries"
-                    );
-                } */
                 await this.join(heads instanceof Entry ? [heads] : heads, {
-                    /* length: amount, */
                     timeout: opts?.fetchEntryTimeout,
-                    cache: {
-                        update: false
-                    }
                 });
             }
         }
@@ -933,7 +814,7 @@ let Log = Log_1 = class Log {
         await log.join(!Array.isArray(entryOrHash) ? [entryOrHash] : entryOrHash, {
             timeout: options.timeout,
             trim: options.trim,
-            verifySignatures: true
+            verifySignatures: true,
         });
         return log;
     }
@@ -943,7 +824,7 @@ let Log = Log_1 = class Log {
      * Finds entries that are the heads of this collection,
      * ie. entries that are not referenced by other entries.
      *
-     * @param {Array<Entry<T>>} entries Entries to search heads from
+     * @param {Array<Entry<T>>} entries - Entries to search heads from
      * @returns {Array<Entry<T>>}
      */
     static findHeads(entries) {