@peerbit/log 1.0.14 → 2.0.0

package/src/heads.ts

@@ -4,34 +4,47 @@ import { HeadsCache } from "./heads-cache.js";
 import { Blocks } from "@peerbit/blocks-interface";
 import { Keychain } from "@peerbit/crypto";
 import { Encoding } from "./encoding.js";
+import { Values } from "./values.js";
+import { logger } from "./logger.js";
+import { EntryIndex } from "./entry-index.js";
 
 export type CacheUpdateOptions = {
 	cache?: { update?: false; reset?: false } | { update: true; reset?: boolean };
 };
 
-interface Config {
+interface Log<T> {
 	storage: Blocks;
 	keychain?: Keychain;
 	memory?: SimpleLevel;
 	encoding: Encoding<any>;
+	entryIndex: EntryIndex<T>;
+	values: Values<T>;
 }
 export class HeadsIndex<T> {
 	private _id: Uint8Array;
 	private _index: Set<string> = new Set();
-	private _gids: Map<string, number>;
+	private _gids: Map<string, Map<string, Entry<T>>>; // gid -> hash -> entry
 	private _headsCache: HeadsCache<T> | undefined;
-	private _config: Config;
+	private _config: Log<T>;
+	private _onGidRemoved?: (gid: string[]) => Promise<void> | void;
 	constructor(id: Uint8Array) {
 		this._gids = new Map();
 		this._id = id;
 	}
 
-	async init(config: Config, options: { entries?: Entry<T>[] } = {}) {
-		this._config = config;
+	async init(
+		log: Log<T>,
+		options: {
+			entries?: Entry<T>[];
+			onGidRemoved?: (gid: string[]) => Promise<void> | void;
+		} = {}
+	) {
+		this._config = log;
+		this._onGidRemoved = options.onGidRemoved;
 		await this.reset(options?.entries || []);
-		if (config.memory) {
+		if (log.memory) {
 			this._headsCache = new HeadsCache(this);
-			return this._headsCache.init(await config.memory.sublevel("heads"));
+			return this._headsCache.init(await log.memory.sublevel("heads"));
 		}
 	}
 
@@ -41,7 +54,7 @@ export class HeadsIndex<T> {
 			replicate?: boolean;
 			reload?: boolean;
 		} & CacheUpdateOptions
-	) {
+	): Promise<Entry<T>[] | undefined> {
 		if (!this._headsCache || (this._headsCache.loaded && !options?.reload)) {
 			return;
 		}
@@ -51,25 +64,19 @@ export class HeadsIndex<T> {
 		if (!heads) {
 			return;
 		}
-		try {
-			const entries = await Promise.all(
-				heads.map(async (x) => {
-					const entry = await Entry.fromMultihash<T>(
-						this._config.storage,
-						x,
-						options
-					);
-					entry.init(this._config);
-					await entry.getGid(); // decrypt gid
-					return entry;
-				})
-			);
-			await this.reset(entries);
-			return entries;
-		} catch (error) {
-			const q = 123;
-			throw error;
-		}
+		const entries = await Promise.all(
+			heads.map(async (x) => {
+				const entry = await this._config.entryIndex.get(x, { load: true });
+				if (!entry) {
+					logger.error("Failed to load entry from head with hash: " + x);
+					return;
+				}
+				await entry.getMeta(); // TODO types,decrypt gid
+				return entry;
+			})
+		);
+		await this.reset(entries.filter((x) => !!x) as Entry<any>[]);
+		return entries as Entry<any>[];
 	}
 
 	get headsCache(): HeadsCache<T> | undefined {
@@ -92,7 +99,7 @@ export class HeadsIndex<T> {
 		return this._index;
 	}
 
-	get gids(): Map<string, number> {
+	get gids(): Map<string, Map<string, Entry<T>>> {
 		return this._gids;
 	}
 
@@ -105,10 +112,16 @@ export class HeadsIndex<T> {
 		options: CacheUpdateOptions = { cache: { reset: true, update: true } }
 	) {
 		this._index.clear();
+		const gidKeys = [...this._gids.keys()];
+
 		this._gids = new Map();
-		if (entries) {
+		if (entries?.length > 0) {
 			await this.putAll(entries, options); // reset cache = true
 		}
+
+		if (gidKeys.length > 0) {
+			this._onGidRemoved?.(gidKeys);
+		}
 	}
 
 	has(cid: string) {
@@ -116,14 +129,14 @@ export class HeadsIndex<T> {
 	}
 
 	async put(entry: Entry<T>, options?: CacheUpdateOptions) {
-		this._putOne(entry);
+		await this._putOne(entry);
 		if (options?.cache?.update) {
 			await this._headsCache?.queue({ added: [entry] }, options.cache.reset);
 		}
 	}
 
 	async putAll(entries: Entry<T>[], options?: CacheUpdateOptions) {
-		this._putAll(entries);
+		await this._putAll(entries);
 		if (options?.cache?.update) {
 			await this._headsCache?.queue({ added: entries }, options.cache.reset);
 		}
@@ -145,7 +158,7 @@ export class HeadsIndex<T> {
 		await this._headsCache?.queue(change, reset);
 	}
 
-	private _putOne(entry: Entry<T>) {
+	private async _putOne(entry: Entry<T>) {
 		if (!entry.hash) {
 			throw new Error("Missing hash");
 		}
@@ -154,44 +167,53 @@ export class HeadsIndex<T> {
 		}
 
 		this._index.add(entry.hash);
-		if (!this._gids.has(entry.gid)) {
-			this._gids.set(entry.gid, 1);
+		const map = this._gids.get(entry.meta.gid);
+		if (!map) {
+			const newMap = new Map();
+			this._gids.set(entry.meta.gid, newMap);
+			newMap.set(entry.hash, entry);
 		} else {
-			this._gids.set(entry.gid, this._gids.get(entry.gid)! + 1);
+			map.set(entry.hash, entry);
+		}
+
+		for (const next of entry.next) {
+			const indexedEntry = this._config.entryIndex.getShallow(next);
+			if (indexedEntry) {
+				await this.del(indexedEntry);
+			}
 		}
 	}
 
-	private _putAll(entries: Entry<T>[]) {
+	private async _putAll(entries: Entry<T>[]) {
 		for (const entry of entries) {
-			this._putOne(entry);
+			await this._putOne(entry);
 		}
 	}
 
 	async del(
-		entry: { hash: string; gid: string },
+		entry: { hash: string; meta: { gid: string } },
 		options?: CacheUpdateOptions
-	): Promise<{
-		removed: boolean;
-		lastWithGid: boolean;
-	}> {
+	): Promise<boolean> {
 		const wasHead = this._index.delete(entry.hash);
 		if (!wasHead) {
-			return {
-				lastWithGid: false,
-				removed: false,
-			};
-		}
-		const newValue = this._gids.get(entry.gid)! - 1;
-		const lastWithGid = newValue <= 0;
-		if (newValue <= 0) {
-			this._gids.delete(entry.gid);
-		} else {
-			this._gids.set(entry.gid, newValue);
+			return false;
+		}
+		let removedGids: Set<string> | undefined = undefined;
+		const map = this._gids.get(entry.meta.gid)!;
+		map.delete(entry.hash);
+		if (map.size <= 0) {
+			this._gids.delete(entry.meta.gid);
+			(removedGids || (removedGids = new Set<string>())).add(entry.meta.gid);
 		}
+
 		if (!entry.hash) {
 			throw new Error("Missing hash");
 		}
 
+		if (removedGids) {
+			await this._onGidRemoved?.([...removedGids]);
+		}
+
 		if (wasHead && options?.cache?.update) {
 			await this._headsCache?.queue(
 				{ removed: [entry.hash] },
@@ -199,10 +221,7 @@ export class HeadsIndex<T> {
 			);
 		}
 
-		return {
-			removed: wasHead,
-			lastWithGid: lastWithGid,
-		};
+		return wasHead;
 		//     this._headsCache = undefined; // TODO do smarter things here, only remove the element needed (?)
 	}
 }

package/src/log-sorting.ts

@@ -1,13 +1,13 @@
-import { Entry } from "./entry.js";
+import { Entry, ShallowEntry } from "./entry.js";
 import { LamportClock as Clock } from "./clock.js";
 import { compare } from "@peerbit/uint8arrays";
 
 const First = (a: any, b: any) => a;
 
 export type ISortFunction = <T>(
-	a: Entry<T>,
-	b: Entry<T>,
-	resolveConflict?: (a: Entry<T>, b: Entry<T>) => number
+	a: ShallowEntry,
+	b: ShallowEntry,
+	resolveConflict?: (a: ShallowEntry, b: ShallowEntry) => number
 ) => number;
 /**
  * Sort two entries as Last-Write-Wins (LWW).
@@ -19,13 +19,17 @@ export type ISortFunction = <T>(
  * @param {Entry} b Second entry
  * @returns {number} 1 if a is latest, -1 if b is latest
  */
-export const LastWriteWins: ISortFunction = <T>(a: Entry<T>, b: Entry<T>) => {
+export const LastWriteWins: ISortFunction = <T>(
+	a: ShallowEntry,
+	b: ShallowEntry
+) => {
 	// Ultimate conflict resolution (take the first/left arg)
 	// Sort two entries by their clock id, if the same always take the first
-	const sortById = (a: Entry<T>, b: Entry<T>) => SortByClockId(a, b, First);
+	const sortById = (a: ShallowEntry, b: ShallowEntry) =>
+		SortByClockId(a, b, First);
 	// Sort two entries by their clock time, if concurrent,
 	// determine sorting using provided conflict resolution function
-	const sortByEntryClocks = (a: Entry<T>, b: Entry<T>) =>
+	const sortByEntryClocks = (a: ShallowEntry, b: ShallowEntry) =>
 		SortByClocks(a, b, sortById);
 	// Sort entries by clock time as the primary sort criteria
 	return sortByEntryClocks(a, b);
@@ -40,14 +44,14 @@ export const LastWriteWins: ISortFunction = <T>(a: Entry<T>, b: Entry<T>) => {
  */
 export const SortByEntryHash: ISortFunction = (a, b) => {
 	// Ultimate conflict resolution (compare hashes)
-	const compareHash = (a: Entry<any>, b: Entry<any>) =>
+	const compareHash = (a: ShallowEntry, b: ShallowEntry) =>
 		a.hash < b.hash ? -1 : 1;
 	// Sort two entries by their clock id, if the same then compare hashes
-	const sortById = (a: Entry<any>, b: Entry<any>) =>
+	const sortById = (a: ShallowEntry, b: ShallowEntry) =>
 		SortByClockId(a, b, compareHash);
 	// Sort two entries by their clock time, if concurrent,
 	// determine sorting using provided conflict resolution function
-	const sortByEntryClocks = (a: Entry<any>, b: Entry<any>) =>
+	const sortByEntryClocks = (a: ShallowEntry, b: ShallowEntry) =>
 		SortByClocks(a, b, sortById);
 	// Sort entries by clock time as the primary sort criteria
 	return sortByEntryClocks(a, b);
@@ -61,12 +65,12 @@ export const SortByEntryHash: ISortFunction = (a, b) => {
  * @returns {number} 1 if a is greater, -1 if b is greater
  */
 export const SortByClocks: ISortFunction = <T>(
-	a: Entry<T>,
-	b: Entry<T>,
-	resolveConflict?: (a: Entry<any>, b: Entry<any>) => number
+	a: ShallowEntry,
+	b: ShallowEntry,
+	resolveConflict?: (a: ShallowEntry, b: ShallowEntry) => number
 ) => {
 	// Compare the clocks
-	const diff = Clock.compare(a.metadata.clock, b.metadata.clock);
+	const diff = Clock.compare(a.meta.clock, b.meta.clock);
 	// If the clocks are concurrent, use the provided
 	// conflict resolution function to determine which comes first
 	return diff === 0 ? (resolveConflict || First)(a, b) : diff;
@@ -82,7 +86,7 @@ export const SortByClocks: ISortFunction = <T>(
 export const SortByClockId: ISortFunction = (a, b, resolveConflict) => {
 	// Sort by ID if clocks are concurrent,
 	// take the entry with a "greater" clock id
-	const clockCompare = compare(a.metadata.clock.id, b.metadata.clock.id);
+	const clockCompare = compare(a.meta.clock.id, b.meta.clock.id);
 	return clockCompare === 0 ? (resolveConflict || First)(a, b) : clockCompare;
 };
 
@@ -93,7 +97,7 @@ export const SortByClockId: ISortFunction = (a, b, resolveConflict) => {
  * @throws {Error} if func ever returns 0
  */
 export const NoZeroes = (func: ISortFunction) => {
-	const comparator = <T>(a: Entry<T>, b: Entry<T>) => {
+	const comparator = <T>(a: ShallowEntry, b: ShallowEntry) => {
 		// Validate by calling the function
 		const result = func(a, b, (a, b) => -1);
 		if (result === 0) {

package/src/log.ts

@@ -12,7 +12,6 @@ import { SimpleLevel } from "@peerbit/lazy-level";
 import { EntryIndex } from "./entry-index.js";
 import * as LogError from "./log-errors.js";
 import * as Sorting from "./log-sorting.js";
-import { isDefined } from "./is-defined.js";
 import { findUniques } from "./find-uniques.js";
 import {
 	EncryptionTemplateMaybeEncrypted,
@@ -42,6 +41,7 @@ const { LastWriteWins, NoZeroes } = Sorting;
 
 export type LogEvents<T> = {
 	onChange?: (change: Change<T>) => void;
+	onGidRemoved?: (gids: string[]) => Promise<void> | void;
 };
 
 export type MemoryProperties = {
@@ -62,19 +62,23 @@ export type LogOptions<T> = LogProperties<T> & LogEvents<T> & MemoryProperties;
 const ENTRY_CACHE_MAX = 1000; // TODO as param
 
 export type AppendOptions<T> = {
-	type?: EntryType;
-	gidSeed?: Uint8Array;
-	nexts?: Entry<any>[];
+	meta?: {
+		type?: EntryType;
+		gidSeed?: Uint8Array;
+		data?: Uint8Array;
+		timestamp?: Timestamp;
+		next?: Entry<any>[];
+	};
+
 	identity?: Identity;
 	signers?: ((
 		data: Uint8Array
 	) => Promise<SignatureWithKey> | SignatureWithKey)[];
-	onGidsShadowed?: (gids: string[]) => void;
+
 	trim?: TrimOptions;
-	timestamp?: Timestamp;
 	encryption?: {
 		keypair: X25519Keypair;
-		reciever: EncryptionTemplateMaybeEncrypted;
+		receiver: EncryptionTemplateMaybeEncrypted;
 	};
 };
 
@@ -113,11 +117,11 @@ export class Log<T> {
 	}
 
 	async open(store: Blocks, identity: Identity, options: LogOptions<T> = {}) {
-		if (!isDefined(store)) {
+		if (store == null) {
 			throw LogError.BlockStoreNotDefinedError();
 		}
 
-		if (!isDefined(identity)) {
+		if (identity == null) {
 			throw new Error("Identity is required");
 		}
 
@@ -125,10 +129,10 @@ export class Log<T> {
 			throw new Error("Already open");
 		}
 
-		const { encoding, trim, keychain, cache } = options;
+		const { encoding, trim, keychain, cache, onGidRemoved } = options;
 		let { sortFn } = options;
 
-		if (!isDefined(sortFn)) {
+		if (sortFn == null) {
 			sortFn = LastWriteWins;
 		}
 		sortFn = sortFn as Sorting.ISortFunction;
@@ -158,7 +162,7 @@ export class Log<T> {
 			throw new Error("Id not set");
 		}
 		this._headsIndex = new HeadsIndex(id);
-		await this._headsIndex.init(this);
+		await this._headsIndex.init(this, { onGidRemoved });
 		this._entryCache = new Cache({ max: ENTRY_CACHE_MAX });
 		this._entryIndex = new EntryIndex({
 			store: this._storage,
@@ -170,15 +174,15 @@ export class Log<T> {
 			{
 				deleteNode: async (node: EntryNode) => {
 					// TODO check if we have before delete?
-					const entry = await this.get(node.value.hash);
+					const entry = await this.get(node.value);
 					//f (!!entry)
 					const a = this.values.length;
 					if (entry) {
 						this.values.deleteNode(node);
-						await this.entryIndex.delete(node.value.hash);
-						await this.headsIndex.del(node.value);
-						this.nextsIndex.delete(node.value.hash);
-						await this.storage.rm(node.value.hash);
+						await this.headsIndex.del(this.entryIndex.getShallow(node.value)!);
+						await this.entryIndex.delete(node.value);
+						this.nextsIndex.delete(node.value);
+						await this.storage.rm(node.value);
 					}
 					const b = this.values.length;
 					if (a === b) {
@@ -243,6 +247,9 @@ export class Log<T> {
 	get values(): Values<T> {
 		return this._values;
 	}
+	get canAppend() {
+		return this._canAppend;
+	}
 
 	/**
 	 * Checks if a entry is part of the log
@@ -506,8 +513,8 @@ export class Log<T> {
 		options: AppendOptions<T> = {}
 	): Promise<{ entry: Entry<T>; removed: Entry<T>[] }> {
 		// Update the clock (find the latest clock)
-		if (options.nexts) {
-			for (const n of options.nexts) {
+		if (options.meta?.next) {
+			for (const n of options.meta.next) {
 				if (!n.hash)
 					throw new Error(
 						"Expecting nexts to already be saved. missing hash for one or more entries"
@@ -517,13 +524,12 @@ export class Log<T> {
 
 		await this.load({ reload: false });
 
-		const hasNext = !!options.nexts; // true for [], which means we have explicitly said that nexts are empty
-		const nexts: Entry<any>[] = options.nexts || (await this.getHeads());
+		const nexts: Entry<any>[] = options.meta?.next || (await this.getHeads());
 
 		// Calculate max time for log/graph
 		const clock = new Clock({
 			id: this._identity.publicKey.bytes,
-			timestamp: options.timestamp || this._hlc.now(),
+			timestamp: options?.meta?.timestamp || this._hlc.now(),
 		});
 
 		const entry = await Entry.create<T>({
@@ -531,23 +537,28 @@ export class Log<T> {
 			identity: options.identity || this._identity,
 			signers: options.signers,
 			data,
-			clock,
-			type: options.type,
+			meta: {
+				clock,
+				type: options.meta?.type,
+				gidSeed: options.meta?.gidSeed,
+				data: options.meta?.data,
+				next: nexts,
+			},
+
 			encoding: this._encoding,
-			next: nexts,
-			gidSeed: options.gidSeed,
+
 			encryption: options.encryption
 				? {
 						keypair: options.encryption.keypair,
-						reciever: {
-							...options.encryption.reciever,
+						receiver: {
+							...options.encryption.receiver,
 						},
 				  }
 				: undefined,
 			canAppend: this._canAppend,
 		});
 
-		if (!isDefined(entry.hash)) {
+		if (!entry.hash) {
 			throw new Error("Unexpected");
 		}
 
@@ -562,35 +573,12 @@ export class Log<T> {
 			}
 		}
 
-		const removedGids: Set<string> = new Set();
-		if (hasNext) {
-			for (const next of nexts) {
-				const deletion = await this._headsIndex.del(next);
-				if (deletion.lastWithGid && next.gid !== entry.gid) {
-					removedGids.add(next.gid);
-				}
-			}
-		} else {
-			// next is all heads, which means we should just overwrite
-			for (const key of this.headsIndex.gids.keys()) {
-				if (key !== entry.gid) {
-					removedGids.add(key);
-				}
-			}
-			await this.headsIndex.reset([entry], { cache: { update: false } });
-		}
-
 		await this._entryIndex.set(entry, false); // save === false, because its already saved when Entry.create
 		await this._headsIndex.put(entry, { cache: { update: false } }); // we will update the cache a few lines later *
 		await this._values.put(entry);
 
 		const removed = await this.processEntry(entry);
 
-		// if next contails all gids
-		if (options.onGidsShadowed && removedGids.size > 0) {
-			options.onGidsShadowed([...removedGids]);
-		}
-
 		entry.init({ encoding: this._encoding, keychain: this._keychain });
 		//	console.log('put entry', entry.hash, (await this._entryIndex._index.size));
 
@@ -718,7 +706,7 @@ export class Log<T> {
 					return;
 				}
 
-				yield next.value.hash;
+				yield next.value;
 				counter++;
 
 				next = nextFn(next);
@@ -782,6 +770,13 @@ export class Log<T> {
 			timeout?: number;
 		} & CacheUpdateOptions
 	): Promise<void> {
+		const definedOptions = {
+			...options,
+			cache: options?.cache ?? {
+				update: true,
+			},
+		};
+
 		await this.load({ reload: false });
 		if (entriesOrLog.length === 0) {
 			return;
@@ -832,7 +827,6 @@ export class Log<T> {
 			}
 		}
 
-		// Resolve missing entries
 		const removedHeads: Entry<T>[] = [];
 		for (const hash of stack) {
 			if (visited.has(hash) || this.has(hash)) {
@@ -851,7 +845,7 @@ export class Log<T> {
 
 			let nexts: string[];
 			if (
-				entry.metadata.type !== EntryType.CUT &&
+				entry.meta.type !== EntryType.CUT &&
 				(nexts = await entry.getNext())
 			) {
 				let isRoot = true;
@@ -888,7 +882,12 @@ export class Log<T> {
 		while (entriesBottomUp.length > 0) {
 			const e = entriesBottomUp.shift()!;
 			await this._joining.get(e.hash);
-			const p = this.joinEntry(e, nextRefs, entriesBottomUp, options).then(
+			const p = this.joinEntry(
+				e,
+				nextRefs,
+				entriesBottomUp,
+				definedOptions
+			).then(
 				() => this._joining.delete(e.hash) // TODO, if head we run into problems with concurrency here!, we add heads at line 929 but resolve here
 			);
 			this._joining.set(e.hash, p);
@@ -910,7 +909,7 @@ export class Log<T> {
 			return;
 		}
 
-		if (!isDefined(e.hash)) {
+		if (!e.hash) {
 			throw new Error("Unexpected");
 		}
 
@@ -929,7 +928,7 @@ export class Log<T> {
 			await this._entryIndex.set(e);
 			await this._values.put(e);
 
-			if (e.metadata.type !== EntryType.CUT) {
+			if (e.meta.type !== EntryType.CUT) {
 				for (const a of e.next) {
 					if (!this.has(a)) {
 						await this.join([a]);
@@ -973,7 +972,7 @@ export class Log<T> {
 	}
 
 	private async processEntry(entry: Entry<T>) {
-		if (entry.metadata.type === EntryType.CUT) {
+		if (entry.meta.type === EntryType.CUT) {
 			return this.deleteRecursively(entry, true);
 		}
 		return [];
@@ -985,6 +984,8 @@ export class Log<T> {
 		const promises: Promise<void>[] = [];
 		let counter = 0;
 		const deleted: Entry<T>[] = [];
+		const deletedGids = new Set();
+
 		while (stack.length > 0) {
 			const entry = stack.pop()!;
 			if ((counter > 0 || !skipFirst) && this.has(entry.hash)) {
@@ -993,6 +994,7 @@ export class Log<T> {
 				await this._values.delete(entry);
 				await this._entryIndex.delete(entry.hash);
 				await this._headsIndex.del(entry);
+
 				this._nextsIndex.delete(entry.hash);
 				deleted.push(entry);
 				promises.push(entry.delete(this._storage));