@peerbit/log 1.0.2

package/src/logger.ts

@@ -0,0 +1,3 @@
+import pino from "pino";
+const logger = pino();
+export { logger };

package/src/snapshot.ts

@@ -0,0 +1,103 @@
+import { Entry } from "./entry.js";
+import { Blocks } from "@peerbit/blocks-interface";
+import {
+	BinaryReader,
+	BinaryWriter,
+	deserialize,
+	field,
+	fixedArray,
+	serialize,
+	variant,
+	vec,
+} from "@dao-xyz/borsh";
+import { waitForAsync } from "@peerbit/time";
+import LocalStore from "@peerbit/lazy-level";
+import { logger } from "./logger.js";
+
+@variant(0)
+export class Snapshot {
+	@field({ type: fixedArray("u8", 32) })
+	id: Uint8Array;
+
+	@field({ type: vec("string") })
+	heads: string[];
+
+	@field({ type: "u64" })
+	size: bigint; // we do a size field, because the "true" log size can be larger then the size of values provided below
+
+	@field({ type: vec(Entry) })
+	values: Entry<any>[];
+
+	constructor(props?: {
+		id: Uint8Array;
+		heads: string[];
+		size: bigint;
+		values: Entry<any>[];
+	}) {
+		if (props) {
+			this.heads = props.heads;
+			this.id = props.id;
+			this.size = props.size;
+			this.values = props.values;
+		}
+	}
+}
+
+export const save = async <T>(
+	snapshotPath: string,
+	blockstore: Blocks,
+	cache: LocalStore,
+	log: {
+		id: Uint8Array;
+		getHeads: () => Promise<string[]>;
+		getValues: () => Promise<Entry<T>[]> | Entry<T>[];
+	}
+): Promise<string> => {
+	const values = await log.getValues();
+	const buf = serialize(
+		new Snapshot({
+			id: log.id,
+			heads: await log.getHeads(),
+			size: BigInt(values.length),
+			values: values,
+		})
+	);
+
+	const snapshot = await blockstore.put(buf);
+	const writer = new BinaryWriter();
+	writer.string(snapshot);
+	await cache.put(snapshotPath, writer.finalize());
+
+	await waitForAsync(() => cache.get(snapshotPath).then((bytes) => !!bytes), {
+		delayInterval: 200,
+		timeout: 10 * 1000,
+	});
+
+	logger.debug(`Saved snapshot: ${snapshot}`);
+	return snapshot;
+};
+
+export const load = async (
+	hash: string,
+	blockstore: Blocks
+): Promise<Snapshot> => {
+	const block = await blockstore.get(hash);
+	if (!block) {
+		throw new Error("Missing snapshot for CID: " + hash);
+	}
+	return deserialize(block, Snapshot);
+};
+
+export const loadFromCache = async (
+	path: string,
+	blockstore: Blocks,
+	cache: LocalStore
+) => {
+	const snapshotOrCID = await cache.get(path);
+	if (!snapshotOrCID) {
+		throw new Error("Missing snapshot CID from local store");
+	}
+	const reader = new BinaryReader(snapshotOrCID);
+	const snapshotCIDString = reader.string();
+	return load(snapshotCIDString, blockstore);
+};

package/src/trim.ts

@@ -0,0 +1,269 @@
+import { Cache } from "@peerbit/cache";
+import PQueue from "p-queue";
+import { Entry } from "./entry.js";
+import { EntryNode, Values } from "./values.js";
+
+const trimOptionsEqual = (a: TrimOptions, b: TrimOptions) => {
+	if (a.type === b.type) {
+		if (a.type === "length" && b.type === "length") {
+			return (
+				a.from === b.from &&
+				a.to === b.to &&
+				a.filter?.canTrim === b.filter?.canTrim
+			);
+		}
+		if (a.type === "bytelength" && b.type === "bytelength") {
+			return (
+				a.from === b.from &&
+				a.to === b.to &&
+				a.filter?.canTrim === b.filter?.canTrim
+			);
+		}
+
+		if (a.type === "time" && b.type === "time") {
+			return a.maxAge === b.maxAge && a.filter?.canTrim === b.filter?.canTrim;
+		}
+	}
+	return false;
+};
+
+const trimOptionsStricter = (from: TrimOptions, to: TrimOptions) => {
+	if (from.type !== to.type || from.filter?.canTrim !== to.filter?.canTrim) {
+		// TODO also check ttl?
+		return true; // we don't really know
+	}
+
+	if (
+		(from.type === "bytelength" || from.type === "length") &&
+		(to.type === "bytelength" || to.type === "length")
+	) {
+		if (from.to > to.to) {
+			return true;
+		}
+		const fromFrom = from.from || from.to;
+		const fromTo = to.from || to.to;
+		return fromFrom > fromTo;
+	} else {
+		if (from.type === "time" && to.type === "time") {
+			return from.maxAge > to.maxAge;
+		}
+	}
+
+	throw new Error("Unexpected");
+};
+
+export type TrimToLengthOption = { type: "length"; to: number; from?: number };
+export type TrimToByteLengthOption = {
+	type: "bytelength";
+	to: number;
+	from?: number;
+};
+
+export type TrimToTime = {
+	type: "time";
+	maxAge: number; // ms
+};
+
+export type TrimCondition =
+	| TrimToByteLengthOption
+	| TrimToLengthOption
+	| TrimToTime;
+export type TrimCanAppendOption = {
+	filter?: {
+		canTrim: (gid: string) => Promise<boolean> | boolean;
+		cacheId?: () => string | number;
+	};
+};
+export type TrimOptions = TrimCanAppendOption & TrimCondition;
+
+interface Log<T> {
+	values: () => Values<T>;
+	deleteNode: (node: EntryNode) => Promise<Entry<T> | undefined>;
+}
+export class Trim<T> {
+	private _trim?: TrimOptions;
+	private _canTrimCacheLastNode: EntryNode | undefined | null;
+	private _trimLastHead: EntryNode | undefined | null;
+	private _trimLastTail: EntryNode | undefined | null;
+	private _trimLastOptions: TrimOptions;
+	private _trimLastSeed: string | number | undefined;
+	private _canTrimCacheHashBreakpoint: Cache<boolean>;
+	private _log: Log<T>;
+	private _queue: PQueue;
+	constructor(log: Log<T>, options?: TrimOptions) {
+		this._log = log;
+		this._trim = options;
+		this._canTrimCacheHashBreakpoint = new Cache({ max: 1e5 });
+		this._queue = new PQueue({ concurrency: 1 });
+	}
+
+	deleteFromCache(entry: Entry<T>) {
+		if (this._canTrimCacheLastNode?.value.hash === entry.hash) {
+			this._canTrimCacheLastNode = this._canTrimCacheLastNode.prev;
+		}
+	}
+
+	get options() {
+		return this._trim;
+	}
+
+	private async trimTask(
+		option: TrimOptions | undefined = this._trim
+	): Promise<Entry<T>[]> {
+		if (!option) {
+			return [];
+		}
+		///  TODO Make this method less ugly
+		const deleted: Entry<T>[] = [];
+
+		let done: () => Promise<boolean> | boolean;
+		const values = this._log.values();
+		if (option.type === "length") {
+			const to = option.to;
+			const from = option.from ?? to;
+			if (values.length < from) {
+				return [];
+			}
+			done = () => values.length <= to;
+		} else if (option.type == "bytelength") {
+			// prune to max sum payload sizes in bytes
+			const byteLengthFrom = option.from ?? option.to;
+
+			if (values.byteLength < byteLengthFrom) {
+				return [];
+			}
+			done = () => values.byteLength <= option.to;
+		} else if (option.type == "time") {
+			const s0 = BigInt(+new Date() * 1e6);
+			const maxAge = option.maxAge * 1e6;
+			done = async () => {
+				if (!values.tail) {
+					return true;
+				}
+
+				const nodeValue = await values.getEntry(values.tail);
+
+				if (!nodeValue) {
+					return true;
+				}
+
+				return s0 - nodeValue.metadata.clock.timestamp.wallTime < maxAge;
+			};
+		} else {
+			return [];
+		}
+
+		const tail = values.tail;
+
+		if (
+			this._trimLastOptions &&
+			trimOptionsStricter(this._trimLastOptions, option)
+		) {
+			this._canTrimCacheHashBreakpoint.clear();
+		}
+
+		const seed = option.filter?.cacheId?.();
+		const cacheProgress = seed != null;
+
+		let changed = false;
+		if (seed !== this._trimLastSeed || !cacheProgress) {
+			// Reset caches
+			this._canTrimCacheHashBreakpoint.clear();
+			this._canTrimCacheLastNode = undefined;
+			changed = true;
+		} else {
+			const trimOptionsChanged =
+				!this._trimLastOptions ||
+				!trimOptionsEqual(this._trimLastOptions, option);
+
+			const changed =
+				this._trimLastHead !== values.head ||
+				this._trimLastTail !== values.tail ||
+				trimOptionsChanged;
+			if (!changed) {
+				return [];
+			}
+		}
+
+		let node: EntryNode | undefined | null = this._canTrimCacheLastNode || tail; // TODO should we do this._canTrimCacheLastNode?.prev instead ?
+		let lastNode: EntryNode | undefined | null = node;
+		let looped = false;
+		const startNode = node;
+		let canTrimByGid: Map<string, boolean> | undefined = undefined;
+
+		// TODO only go through heads?
+		while (
+			node &&
+			!(await done()) &&
+			values.length > 0 &&
+			node &&
+			(!looped || node !== startNode)
+		) {
+			let deleteAble: boolean | undefined = true;
+			if (option.filter?.canTrim) {
+				canTrimByGid = canTrimByGid || new Map();
+				deleteAble = canTrimByGid.get(node.value.gid);
+				if (deleteAble === undefined) {
+					deleteAble = await option.filter?.canTrim(node.value.gid);
+					canTrimByGid.set(node.value.gid, deleteAble);
+				}
+
+				if (!deleteAble && cacheProgress) {
+					// ignore it
+					this._canTrimCacheHashBreakpoint.add(node.value.hash, true);
+				}
+			}
+
+			// Delete, and update current node
+			if (deleteAble) {
+				// Do this before deleteNode, else prev/next might be gone!
+				const prev = node.prev;
+				const next = node.next;
+
+				const entry = await this._log.deleteNode(node);
+				if (entry) {
+					deleted.push(entry);
+				}
+
+				node = prev;
+				// If we don't do this, we might, next time start to iterate from a node that does not exist
+				// we do prev 'or' next because next time we want to start as close as possible to where we left of
+				lastNode = prev || next;
+			} else {
+				lastNode = node;
+				node = node?.prev;
+			}
+
+			if (!node) {
+				if (!looped && changed && !cacheProgress) {
+					node = tail;
+					looped = true;
+				} else {
+					break;
+				}
+			}
+		}
+
+		// remember the node where we started last time from
+		this._canTrimCacheLastNode = node || lastNode;
+		this._trimLastHead = values.head;
+		this._trimLastTail = values.tail;
+		this._trimLastOptions = option;
+		this._trimLastSeed = seed;
+
+		return deleted;
+	}
+	/**
+	 * @param options
+	 * @returns deleted entries
+	 */
+	async trim(
+		option: TrimOptions | undefined = this._trim
+	): Promise<Entry<T>[]> {
+		const result = await this._queue.add(() => this.trimTask(option));
+		if (result instanceof Object) {
+			return result;
+		}
+		throw new Error("Something when wrong when trimming");
+	}
+}

package/src/types.ts

@@ -0,0 +1,12 @@
+import { field, vec } from "@dao-xyz/borsh";
+
+export class StringArray {
+	@field({ type: vec("string") })
+	arr: string[];
+
+	constructor(properties?: { arr: string[] }) {
+		if (properties) {
+			this.arr = properties.arr;
+		}
+	}
+}

package/src/utils.ts

@@ -0,0 +1,2 @@
+export const max = <T>(...args: T[]) => args.reduce((m, e) => (e > m ? e : m));
+export const min = <T>(...args: T[]) => args.reduce((m, e) => (e < m ? e : m));

package/src/values.ts

@@ -0,0 +1,193 @@
+import { Entry } from "./entry";
+import { ISortFunction } from "./log-sorting";
+import yallist from "yallist";
+import { EntryIndex } from "./entry-index";
+
+type Storage<T> = (
+	hash: string
+) => Promise<Entry<T> | undefined> | Entry<T> | undefined;
+
+interface Value {
+	hash: string;
+	gid: string;
+	byteLength: number;
+}
+
+export type EntryNode = yallist.Node<Value>;
+
+export class Values<T> {
+	/**
+	 * Keep track of sorted elements in descending sort order (i.e. newest elements)
+	 */
+	private _values: yallist<Value>;
+	private _sortFn: ISortFunction;
+	private _byteLength: number;
+	private _entryIndex: EntryIndex<T>;
+
+	constructor(
+		entryIndex: EntryIndex<T>,
+		sortFn: ISortFunction,
+		entries: Entry<T>[] = []
+	) {
+		this._values = yallist.create(
+			entries
+				.slice()
+				.sort(sortFn)
+				.reverse()
+				.map((x) => {
+					if (!x.hash) throw new Error("Unexpected");
+					return {
+						hash: x.hash,
+						byteLength: x._payload.byteLength,
+						gid: x.gid,
+					};
+				})
+		);
+		this._byteLength = 0;
+		entries.forEach((entry) => {
+			this._byteLength += entry._payload.byteLength;
+		});
+		this._sortFn = sortFn;
+		this._entryIndex = entryIndex;
+	}
+
+	toArray(): Promise<Entry<T>[]> {
+		return Promise.all(
+			this._values.toArrayReverse().map((x) => this._entryIndex.get(x.hash))
+		).then((arr) => arr.filter((x) => !!x)) as Promise<Entry<T>[]>; // we do reverse because we assume the log is only meaningful if we read it from start to end
+	}
+
+	get head() {
+		return this._values.head;
+	}
+	get tail() {
+		return this._values.tail;
+	}
+	get length() {
+		return this._values.length;
+	}
+
+	private _putPromise: Map<string, Promise<any>> = new Map();
+	async put(value: Entry<T>) {
+		let promise = this._putPromise.get(value.hash);
+		if (promise) {
+			return promise;
+		}
+		promise = this._put(value).then((v) => {
+			this._putPromise.delete(value.hash);
+			return v;
+		});
+		this._putPromise.set(value.hash, promise);
+		return promise;
+	}
+	async _put(value: Entry<T>) {
+		// assume we want to insert at head (or somehere close)
+		let walker = this._values.head;
+		let last: EntryNode | undefined = undefined;
+		while (walker) {
+			const walkerValue = await this.getEntry(walker);
+			if (!walkerValue) {
+				throw new Error("Missing walker value");
+			}
+			if (walkerValue.hash === value.hash) {
+				return; // already exist!
+			}
+
+			if (this._sortFn(walkerValue, value) < 0) {
+				break;
+			}
+			last = walker;
+			walker = walker.next;
+			continue;
+		}
+
+		this._byteLength += value._payload.byteLength;
+		if (!value.hash) {
+			throw new Error("Unexpected");
+		}
+
+		_insertAfter(this._values, last, {
+			byteLength: value._payload.byteLength,
+			gid: value.gid,
+			hash: value.hash,
+		});
+	}
+
+	async delete(value: Entry<T> | string) {
+		const hash = typeof value === "string" ? value : value.hash;
+		// Assume we want to delete at tail (or somwhere close)
+
+		let walker = this._values.tail;
+		while (walker) {
+			const walkerValue = await this.getEntry(walker);
+
+			if (!walkerValue) {
+				throw new Error("Missing walker value");
+			}
+
+			if (walkerValue.hash === hash) {
+				this._values.removeNode(walker);
+				this._byteLength -= walkerValue._payload.byteLength;
+				return;
+			}
+			walker = walker.prev; // prev will be undefined if you do removeNode(walker)
+		}
+		throw new Error(
+			"Failed to delete, entry does not exist" +
+				" ??? " +
+				this.length +
+				" ??? " +
+				hash
+		);
+	}
+
+	deleteNode(node: EntryNode) {
+		this._values.removeNode(node);
+		this._byteLength -= node.value.byteLength;
+		return;
+	}
+
+	pop() {
+		const value = this._values.pop();
+		if (value) {
+			this._byteLength -= value.byteLength;
+		}
+		return value;
+	}
+
+	get byteLength() {
+		return this._byteLength;
+	}
+
+	async getEntry(node: EntryNode) {
+		return this._entryIndex.get(node.value.hash);
+	}
+}
+
+function _insertAfter(
+	self: yallist<any>,
+	node: EntryNode | undefined,
+	value: Value
+) {
+	const inserted = !node
+		? new yallist.Node(
+				value,
+				null as any,
+				self.head as EntryNode | undefined,
+				self
+		  )
+		: new yallist.Node(value, node, node.next as EntryNode | undefined, self);
+
+	// is tail
+	if (inserted.next === null) {
+		self.tail = inserted;
+	}
+
+	// is head
+	if (inserted.prev === null) {
+		self.head = inserted;
+	}
+
+	self.length++;
+	return inserted;
+}