@peerbit/log 3.0.34-aa577a5 → 3.0.34-cccc078

package/src/clock.ts

@@ -22,10 +22,12 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
  */
-
-import { field, variant } from "@dao-xyz/borsh";
-import { compare, equals } from "uint8arrays";
+import {
+	field,
+	/* , variant */
+} from "@dao-xyz/borsh";
 import { hrtime } from "@peerbit/time";
+import { compare, equals } from "uint8arrays";
 
 const hrTimeNow = hrtime.bigint();
 const startTime = BigInt(Date.now()) * BigInt(1e6) - hrTimeNow;
@@ -36,7 +38,7 @@ export function fromBits(low: any, high: any, unsigned: any, target?: any) {
 		return {
 			low: low | 0,
 			high: high | 0,
-			unsigned: !!unsigned
+			unsigned: !!unsigned,
 		};
 	}
 	target.low = low | 0;
@@ -56,7 +58,7 @@ function bigIntCoerce(input: any, fallback: any) {
 	return fallback;
 }
 
-@variant(0)
+/* @variant(0) */
 export class Timestamp {
 	@field({ type: "u64" })
 	wallTime: bigint;
@@ -90,7 +92,7 @@ export class Timestamp {
 	clone(): Timestamp {
 		return new Timestamp({
 			wallTime: this.wallTime,
-			logical: this.logical
+			logical: this.logical,
 		});
 	}
 
@@ -112,14 +114,14 @@ export class HLC {
 			wallTimeUpperBound?: bigint;
 			toleratedForwardClockJump?: bigint;
 			last?: Timestamp;
-		} = {}
+		} = {},
 	) {
 		this.wallTime = properties.wallTime || bigintTime;
 		this.maxOffset = bigIntCoerce(properties.maxOffset, 0n);
 		this.wallTimeUpperBound = bigIntCoerce(properties.wallTimeUpperBound, 0n);
 		this.toleratedForwardClockJump = bigIntCoerce(
 			properties.toleratedForwardClockJump,
-			0n
+			0n,
 		);
 		this.last = new Timestamp({ wallTime: this.wallTime() });
 		if (properties.last) {
@@ -174,9 +176,11 @@ export class ClockOffsetError extends Error {
 	maxOffset: bigint;
 	constructor(offset: bigint, maxOffset: bigint) {
 		super(
-			`The received time is ${offset / n1e6
-			}ms ahead of the wall time, exceeding the 'maxOffset' limit of ${maxOffset / n1e6
-			}ms.`
+			`The received time is ${
+				offset / n1e6
+			}ms ahead of the wall time, exceeding the 'maxOffset' limit of ${
+				maxOffset / n1e6
+			}ms.`,
 		);
 		this.offset = offset;
 		this.maxOffset = maxOffset;
@@ -188,8 +192,9 @@ export class WallTimeOverflowError extends Error {
 	maxTime: bigint;
 	constructor(time: bigint, maxTime: bigint) {
 		super(
-			`The wall time ${time / n1e6}ms exceeds the max time of ${maxTime / n1e6
-			}ms.`
+			`The wall time ${time / n1e6}ms exceeds the max time of ${
+				maxTime / n1e6
+			}ms.`,
 		);
 		this.time = time;
 		this.maxTime = maxTime;
@@ -201,15 +206,16 @@ export class ForwardJumpError extends Error {
 	tolerance: bigint;
 	constructor(timejump: bigint, tolerance: bigint) {
 		super(
-			`Detected a forward time jump of ${timejump / n1e6
-			}ms, which exceed the allowed tolerance of ${tolerance / n1e6}ms.`
+			`Detected a forward time jump of ${
+				timejump / n1e6
+			}ms, which exceed the allowed tolerance of ${tolerance / n1e6}ms.`,
 		);
 		this.timejump = timejump;
 		this.tolerance = tolerance;
 	}
 }
 
-@variant(0)
+/* @variant(0) */
 export class LamportClock {
 	@field({ type: Uint8Array })
 	id: Uint8Array;
@@ -222,13 +228,13 @@ export class LamportClock {
 		if (!properties.timestamp) {
 			this.timestamp = new Timestamp({
 				wallTime: bigintTime(),
-				logical: 0
+				logical: 0,
 			});
 		} else {
 			if (typeof properties.timestamp === "number") {
 				this.timestamp = new Timestamp({
 					wallTime: bigintTime(),
-					logical: properties.timestamp
+					logical: properties.timestamp,
 				});
 			} else {
 				this.timestamp = properties.timestamp;
@@ -239,7 +245,7 @@ export class LamportClock {
 	clone() {
 		return new LamportClock({
 			id: this.id,
-			timestamp: this.timestamp.clone()
+			timestamp: this.timestamp.clone(),
 		});
 	}
 

package/src/encoding.ts

@@ -9,19 +9,19 @@ export const NO_ENCODING: Encoding<any> = {
 		if (obj instanceof Uint8Array === false) {
 			throw new Error(
 				"With NO_ENCODING only Uint8arrays are allowed, received: " +
-				(obj?.["constructor"]?.["name"] || typeof obj)
+					(obj?.["constructor"]?.["name"] || typeof obj),
 			);
 		}
 		return obj;
 	},
 	decoder: (bytes: Uint8Array) => {
 		return bytes;
-	}
+	},
 };
 
 export const BORSH_ENCODING = <T>(clazz: AbstractType<T>): Encoding<T> => {
 	return {
 		decoder: (bytes: Uint8Array) => deserialize(bytes, clazz),
-		encoder: (data: any) => serialize(data)
+		encoder: (data: any) => serialize(data),
 	};
 };

package/src/entry-index.ts

@@ -1,36 +1,291 @@
+import { deserialize, serialize } from "@dao-xyz/borsh";
+import { type Blocks } from "@peerbit/blocks-interface";
 import { Cache } from "@peerbit/cache";
-import { Entry, type ShallowEntry } from "./entry.js";
-import { deserialize } from "@dao-xyz/borsh";
+import type { PublicSignKey } from "@peerbit/crypto";
+import {
+	BoolQuery,
+	CountRequest,
+	DeleteRequest,
+	type Index,
+	Not,
+	Or,
+	type Query,
+	SearchRequest,
+	type Shape,
+	Sort,
+	SortDirection,
+	StringMatch,
+	StringMatchMethod,
+	SumRequest,
+	iterate,
+	toId,
+} from "@peerbit/indexer-interface";
+import {
+	Entry,
+	EntryType,
+	type ShallowEntry,
+	type ShallowOrFullEntry,
+} from "./entry.js";
+import type { SortFn } from "./log-sorting.js";
 import { logger } from "./logger.js";
-import { type Blocks } from "@peerbit/blocks-interface";
+
+export type ResultsIterator<T> = {
+	close: () => void | Promise<void>;
+	next: (number: number) => T[] | Promise<T[]>;
+	done: () => boolean;
+	all(): T[] | Promise<T[]>;
+};
+
+const ENTRY_CACHE_MAX_SIZE = 10; // TODO as param for log
+
+type ResolveFullyOptions =
+	| true
+	| {
+			type: "full";
+			replicate?: boolean;
+			signal?: AbortSignal;
+			timeout?: number;
+			ignoreMissing?: boolean;
+	  };
+type ResolveShapeOptions = { type: "shape"; shape: Shape };
+export type MaybeResolveOptions =
+	| false
+	| ResolveFullyOptions
+	| ResolveShapeOptions;
+export type ReturnTypeFromResolveOptions<
+	R extends MaybeResolveOptions,
+	T,
+> = R extends false | undefined
+	? ShallowEntry
+	: R extends { type: "shape" }
+		? any
+		: Entry<T>;
 
 export class EntryIndex<T> {
-	_cache: Cache<Entry<T> | null>;
-	_blocks: Blocks;
-	_init: (entry: Entry<T>) => void;
-	_index: Map<string, ShallowEntry>;
-
-	constructor(properties: {
-		store: Blocks;
-		init: (entry: Entry<T>) => void;
-		cache: Cache<Entry<T>>;
-	}) {
-		this._cache = properties.cache;
-		this._blocks = properties.store;
-		this._init = properties.init;
-		this._index = new Map();
-	}
-
-	async set(v: Entry<T>, toMultihash = true) {
-		if (toMultihash) {
-			const existingHash = v.hash;
-			v.hash = undefined as any;
+	private cache: Cache<Entry<T>>;
+	private sortReversed: Sort[];
+	private initialied = false;
+	private _length: number;
+	private insertionPromises: Map<string, Promise<void>>;
+	constructor(
+		readonly properties: {
+			store: Blocks;
+			publicKey: PublicSignKey;
+			init: (entry: Entry<T>) => void;
+			cache?: Cache<Entry<T>>;
+			index: Index<ShallowEntry>;
+			sort: SortFn;
+			onGidRemoved?: (gid: string[]) => Promise<void> | void;
+		},
+	) {
+		this.sortReversed = properties.sort.sort.map((x) =>
+			deserialize(serialize(x), Sort),
+		);
+		this.sortReversed.map(
+			(x) =>
+				(x.direction =
+					x.direction === SortDirection.DESC
+						? SortDirection.ASC
+						: SortDirection.DESC),
+		);
+		this.cache = properties.cache ?? new Cache({ max: ENTRY_CACHE_MAX_SIZE });
+		this._length = 0;
+		this.insertionPromises = new Map();
+	}
+
+	getHeads<R extends MaybeResolveOptions = false>(
+		gid?: string,
+		resolve: R = false as R,
+	): ResultsIterator<ReturnTypeFromResolveOptions<R, T>> {
+		const query: Query[] = [];
+		query.push(new BoolQuery({ key: "head", value: true }));
+		if (gid) {
+			query.push(
+				new StringMatch({
+					key: ["meta", "gid"],
+					value: gid,
+					caseInsensitive: false,
+					method: StringMatchMethod.exact,
+				}),
+			);
+		}
+		return this.query(query, undefined, resolve);
+	}
+
+	getHasNext<R extends MaybeResolveOptions>(
+		next: string,
+		resolve?: R,
+	): ResultsIterator<ReturnTypeFromResolveOptions<R, T>> {
+		const query: Query[] = [
+			new StringMatch({
+				key: ["meta", "next"],
+				value: next,
+				caseInsensitive: false,
+				method: StringMatchMethod.exact,
+			}),
+		];
+		return this.query(query, undefined, resolve);
+	}
+
+	countHasNext(next: string, excludeHash: string | undefined = undefined) {
+		const query: Query[] = [
+			new StringMatch({
+				key: ["meta", "next"],
+				value: next,
+				caseInsensitive: false,
+				method: StringMatchMethod.exact,
+			}),
+		];
+		if (excludeHash) {
+			query.push(
+				new Not(
+					new StringMatch({
+						key: ["hash"],
+						value: excludeHash,
+						caseInsensitive: false,
+						method: StringMatchMethod.exact,
+					}),
+				),
+			);
+		}
+		return this.properties.index.count(new CountRequest({ query }));
+	}
+
+	query<R extends MaybeResolveOptions>(
+		query: Query[],
+		sort = this.properties.sort.sort,
+		options?: R,
+	): ResultsIterator<ReturnTypeFromResolveOptions<R, T>> {
+		const iterator = iterate(
+			this.properties.index,
+			new SearchRequest({ query, sort }),
+		);
+		let resolveInFull = options
+			? options === true
+				? true
+				: options.type === "full"
+			: false;
+		let resolveInFullOptions: ResolveFullyOptions | undefined = resolveInFull
+			? (options as ResolveFullyOptions)
+			: undefined;
+		let nextShape = resolveInFull
+			? ({ hash: true } as const)
+			: ((options as { shape: Shape })?.shape as Shape);
+
+		const next = async (
+			amount: number,
+		): Promise<ReturnTypeFromResolveOptions<R, T>[]> => {
+			const results = await iterator.next(amount, { shape: nextShape });
+			if (resolveInFull) {
+				const maybeResolved = await Promise.all(
+					results.results.map((x) =>
+						this.resolve(x.value.hash, resolveInFullOptions),
+					),
+				);
+				return maybeResolved.filter((x) => !!x) as ReturnTypeFromResolveOptions<
+					R,
+					T
+				>[];
+			} else {
+				return results.results.map(
+					(x) => x.value,
+				) as ReturnTypeFromResolveOptions<R, T>[];
+			}
+		};
+
+		return {
+			close: iterator.close,
+			done: iterator.done,
+			next,
+			all: async () => {
+				const results: ReturnTypeFromResolveOptions<R, T>[] = [];
+				while (!iterator.done()) {
+					for (const element of await next(100)) {
+						results.push(element);
+					}
+				}
+				await iterator.close();
+				return results;
+			},
+		};
+	}
+
+	async getOldest<
+		T extends boolean,
+		R = T extends true ? Entry<any> : ShallowEntry,
+	>(resolve?: T): Promise<R | undefined> {
+		const iterator = this.query([], this.properties.sort.sort, resolve);
+		const results = await iterator.next(1);
+		await iterator.close();
+		return results[0] as R;
+	}
+
+	async getNewest<
+		T extends boolean,
+		R = T extends true ? Entry<any> : ShallowEntry,
+	>(resolve?: T): Promise<R | undefined> {
+		const iterator = this.query([], this.sortReversed, resolve);
+		const results = await iterator.next(1);
+		await iterator.close();
+		return results[0] as R;
+	}
+
+	async getBefore<
+		T extends boolean,
+		R = T extends true ? Entry<any> : ShallowEntry,
+	>(before: ShallowOrFullEntry<any>, resolve?: T): Promise<R | undefined> {
+		const iterator = this.query(
+			this.properties.sort.before(before),
+			this.sortReversed,
+			resolve,
+		);
+		const results = await iterator.next(1);
+		await iterator.close();
+		return results[0] as R;
+	}
+	async getAfter<
+		T extends boolean,
+		R = T extends true ? Entry<any> : ShallowEntry,
+	>(before: ShallowOrFullEntry<any>, resolve?: T): Promise<R | undefined> {
+		const iterator = this.query(
+			this.properties.sort.after(before),
+			this.properties.sort.sort,
+			resolve,
+		);
+		const results = await iterator.next(1);
+		await iterator.close();
+
+		return results[0] as R;
+	}
+
+	async get(k: string, options?: ResolveFullyOptions) {
+		return this.resolve(k, options);
+	}
+
+	async getShallow(k: string) {
+		return this.properties.index.get(toId(k));
+	}
+
+	async has(k: string) {
+		const result = await this.properties.index.get(toId(k), {
+			shape: { hash: true },
+		});
+		return result != null;
+	}
+
+	async put(
+		entry: Entry<any>,
+		properties: { unique: boolean; isHead: boolean; toMultiHash: boolean },
+	) {
+		if (properties.toMultiHash) {
+			const existingHash = entry.hash;
+			entry.hash = undefined as any;
 			try {
-				const hash = await Entry.toMultihash(this._blocks, v);
-				v.hash = existingHash;
-				if (v.hash === undefined) {
-					v.hash = hash; // can happen if you sync entries that you load directly from ipfs
-				} else if (existingHash !== v.hash) {
+				const hash = await Entry.toMultihash(this.properties.store, entry);
+				entry.hash = existingHash;
+				if (entry.hash === undefined) {
+					entry.hash = hash; // can happen if you sync entries that you load directly from ipfs
+				} else if (existingHash !== entry.hash) {
 					logger.error("Head hash didn't match the contents");
 					throw new Error("Head hash didn't match the contents");
 				}
@@ -38,42 +293,192 @@ export class EntryIndex<T> {
 				logger.error(error);
 				throw error;
 			}
+		} else {
+			if (!entry.hash) {
+				throw new Error("Missing hash");
+			}
+		}
+
+		const existingPromise = this.insertionPromises.get(entry.hash);
+		if (existingPromise) {
+			return existingPromise;
+		} else {
+			const fn = async () => {
+				this.cache.add(entry.hash, entry);
+
+				if (properties.unique === true || !(await this.has(entry.hash))) {
+					this._length++;
+				}
+
+				await this.properties.index.put(entry.toShallow(properties.isHead));
+
+				// check if gids has been shadowed, by query all nexts that have a different gid
+				if (this.properties.onGidRemoved && entry.meta.next.length > 0) {
+					let nextMatches: Query[] = [];
+
+					for (const next of entry.meta.next) {
+						nextMatches.push(
+							new StringMatch({
+								key: ["hash"],
+								value: next,
+								caseInsensitive: false,
+								method: StringMatchMethod.exact,
+							}),
+						);
+					}
+
+					const nextsWithOthersGids: { hash: string; meta: { gid: string } }[] =
+						await this.query(
+							[
+								new Or(nextMatches),
+								new Not(
+									new StringMatch({
+										key: ["meta", "gid"],
+										value: entry.meta.gid,
+									}),
+								),
+							],
+							undefined,
+							{ type: "shape", shape: { hash: true, meta: { gid: true } } },
+						).all();
+
+					let shadowedGids = new Set<string>();
+					for (const next of nextsWithOthersGids) {
+						// check that this entry is not referenced by other
+						const nexts = await this.countHasNext(next.hash, entry.hash);
+						if (nexts > 0) {
+							continue;
+						}
+						shadowedGids.add(next.meta.gid);
+					}
+
+					if (shadowedGids.size > 0) {
+						this.properties.onGidRemoved?.([...shadowedGids]);
+					}
+				}
+
+				// mark all next entries as not heads
+				await this.privateUpdateNextHeadProperty(entry, false);
+
+				this.insertionPromises.delete(entry.hash);
+			};
+			const promise = fn();
+			this.insertionPromises.set(entry.hash, promise);
+			return promise;
+		}
+	}
+
+	async delete(k: string) {
+		this.cache.del(k);
+
+		let shallow = (await this.getShallow(k))?.value;
+		if (!shallow) {
+			return; // already deleted
+		}
+
+		let deleted = await this.properties.index.del(
+			new DeleteRequest({ query: { hash: k } }),
+		);
+		await this.properties.store.rm(k);
+
+		if (deleted.length > 0) {
+			this._length -= deleted.length;
+
+			// mark all next entries as new heads
+			await this.privateUpdateNextHeadProperty(shallow, true);
+			return shallow;
+		}
+	}
+
+	async getMemoryUsage() {
+		return this.properties.index.sum(new SumRequest({ key: "payloadSize" }));
+	}
+
+	private async privateUpdateNextHeadProperty(
+		from: ShallowEntry | Entry<any>,
+		isHead: boolean,
+	) {
+		if (from.meta.type === EntryType.CUT) {
+			// if the next is a cut, we can't update it, since it's not in the index
+			return;
+		}
+
+		for (const next of from.meta.next) {
+			const indexedEntry = await this.properties.index.get(toId(next));
+
+			if (!indexedEntry) {
+				continue; // we could end up here because another entry with same next ref is of CUT and has removed it from the index
+			}
+
+			if (isHead) {
+				const noPointersToNext = (await this.countHasNext(next)) === 0;
+				if (noPointersToNext) {
+					indexedEntry.value.head = true;
+					if (indexedEntry) {
+						await this.properties.index.put(indexedEntry.value);
+					}
+				}
+			} else {
+				indexedEntry.value.head = false;
+				if (indexedEntry) {
+					await this.properties.index.put(indexedEntry.value);
+				}
+			}
 		}
-		this._cache.add(v.hash, v);
-		this._index.set(v.hash, v.toShallow());
 	}
-	has(k: string) {
-		return this._index.has(k);
+
+	async clear() {
+		const iterator = await this.query([], undefined, false);
+		while (!iterator.done()) {
+			const results = await iterator.next(100);
+			for (const result of results) {
+				await this.delete(result.hash);
+			}
+		}
+		await this.properties.index.drop();
+		await this.properties.index.start();
+		this.cache.clear();
 	}
 
-	async get(
+	get length() {
+		if (!this.initialied) {
+			throw new Error("Not initialized");
+		}
+		return this._length;
+	}
+
+	async init() {
+		this._length = await this.properties.index.getSize();
+		this.initialied = true;
+	}
+
+	private async resolve(
 		k: string,
-		options?: { load?: boolean; replicate?: boolean; timeout?: number }
+		options?: ResolveFullyOptions,
 	): Promise<Entry<T> | undefined> {
-		if (this._index.has(k) || options?.load) {
-			let mem = this._cache.get(k);
+		let coercedOptions = typeof options === "object" ? options : undefined;
+		if (await this.has(k)) {
+			let mem = this.cache.get(k);
 			if (mem === undefined) {
-				mem = await this.getFromStore(k, options);
+				mem = await this.resolveFromStore(k, coercedOptions);
 				if (mem) {
-					this._init(mem);
+					this.properties.init(mem);
 					mem.hash = k;
+				} else if (coercedOptions?.ignoreMissing !== true) {
+					throw new Error("Failed to load entry from head with hash: " + k);
 				}
-				this._cache.add(k, mem);
+				this.cache.add(k, mem ?? undefined);
 			}
 			return mem ? mem : undefined;
 		}
 		return undefined;
 	}
 
-	getShallow(k: string) {
-		return this._index.get(k);
-	}
-
-	private async getFromStore(
+	private async resolveFromStore(
 		k: string,
-		options?: { replicate?: boolean; timeout?: number }
+		options?: { signal?: AbortSignal; replicate?: boolean; timeout?: number },
 	): Promise<Entry<T> | null> {
-		const value = await this._blocks.get(k, options);
+		const value = await this.properties.store.get(k, options);
 		if (value) {
 			const entry = deserialize(value, Entry);
 			entry.size = value.length;
@@ -81,10 +486,4 @@ export class EntryIndex<T> {
 		}
 		return null;
 	}
-
-	async delete(k: string) {
-		this._cache.del(k);
-		this._index.delete(k);
-		return this._blocks.rm(k);
-	}
 }

package/src/entry-with-refs.ts

@@ -1,4 +1,4 @@
-import { Entry } from "./entry.js";
+import type { Entry } from "./entry.js";
 
 export interface EntryWithRefs<T> {
 	entry: Entry<T>;