@peerbit/log 3.0.34 → 4.0.0-55cebfe

package/src/__benchmark__/index.ts

@@ -1,130 +0,0 @@
-import B from "benchmark";
-import { field, option, serialize, variant } from "@dao-xyz/borsh";
-import { v4 as uuid } from "uuid";
-import crypto from "crypto";
-import { Log } from "../log.js";
-import { AnyBlockStore } from "@peerbit/blocks";
-import { Ed25519Keypair } from "@peerbit/crypto";
-import { BORSH_ENCODING } from "../encoding.js";
-import { Entry } from "../entry.js";
-
-// Run with "node --loader ts-node/esm ./src/__benchmark__/index.ts"
-
-@variant("document")
-class Document {
-	@field({ type: "string" })
-	id: string;
-
-	@field({ type: option("string") })
-	name?: string;
-
-	@field({ type: option("u64") })
-	number?: bigint;
-
-	@field({ type: Uint8Array })
-	bytes: Uint8Array;
-
-	constructor(opts: Document) {
-		if (opts) {
-			this.id = opts.id;
-			this.name = opts.name;
-			this.number = opts.number;
-			this.bytes = opts.bytes;
-		}
-	}
-}
-let log: Log<Document>;
-let store: AnyBlockStore;
-const key = await Ed25519Keypair.create();
-
-const reset = async () => {
-	log = new Log<Document>();
-	store = new AnyBlockStore();
-	await log.open(store, key, { encoding: BORSH_ENCODING(Document) });
-};
-await reset();
-
-class NestedEntry {
-	@field({ type: Entry })
-	entry: Entry<any>;
-
-	constructor(entry: Entry<any>) {
-		this.entry = entry;
-	}
-}
-const suite = new B.Suite({ delay: 100 });
-suite
-	.add("1e3", {
-		fn: async (deferred) => {
-			const doc = new Document({
-				id: uuid(),
-				name: "hello",
-				number: 1n,
-				bytes: crypto.randomBytes(1e3)
-			});
-			await log.append(doc);
-
-			deferred.resolve();
-		},
-		defer: true
-	})
-	.add("1e4", {
-		fn: async (deferred) => {
-			const doc = new Document({
-				id: uuid(),
-				name: "hello",
-				number: 1n,
-				bytes: crypto.randomBytes(1e4)
-			});
-			await log.append(doc);
-
-			deferred.resolve();
-		},
-		defer: true
-	})
-	.add("1e5", {
-		fn: async (deferred) => {
-			const doc = new Document({
-				id: uuid(),
-				name: "hello",
-				number: 1n,
-				bytes: crypto.randomBytes(1e5)
-			});
-			const entry = await log.append(doc);
-			serialize(new NestedEntry(entry.entry));
-			serialize(new NestedEntry(entry.entry));
-			serialize(new NestedEntry(entry.entry));
-			serialize(new NestedEntry(entry.entry));
-			serialize(new NestedEntry(entry.entry));
-			serialize(new NestedEntry(entry.entry));
-			serialize(new NestedEntry(entry.entry));
-			serialize(new NestedEntry(entry.entry));
-			serialize(new NestedEntry(entry.entry));
-			serialize(new NestedEntry(entry.entry));
-
-			deferred.resolve();
-		},
-		defer: true
-	})
-	.add("1e6", {
-		fn: async (deferred) => {
-			const doc = new Document({
-				id: uuid(),
-				name: "hello",
-				number: 1n,
-				bytes: crypto.randomBytes(1e5)
-			});
-			await log.append(doc);
-
-			deferred.resolve();
-		},
-		defer: true
-	})
-	.on("cycle", async (event: any) => {
-		console.log(String(event.target));
-		await reset();
-	})
-	.on("error", (err) => {
-		throw err;
-	})
-	.run();

package/src/heads.ts

@@ -1,233 +0,0 @@
-import { Entry } from "./entry.js";
-import { AnyStore } from "@peerbit/any-store";
-import { HeadsCache } from "./heads-cache.js";
-import { Blocks } from "@peerbit/blocks-interface";
-import { Keychain } from "@peerbit/keychain";
-import { Encoding } from "./encoding.js";
-import { Values } from "./values.js";
-import { logger } from "./logger.js";
-import { EntryIndex } from "./entry-index.js";
-
-export type CacheUpdateOptions = {
-	cache?: { update?: false; reset?: false } | { update: true; reset?: boolean };
-};
-
-interface Log<T> {
-	blocks: Blocks;
-	keychain?: Keychain;
-	memory?: AnyStore;
-	encoding: Encoding<any>;
-	entryIndex: EntryIndex<T>;
-	values: Values<T>;
-}
-export class HeadsIndex<T> {
-	private _id: Uint8Array;
-	private _index: Set<string> = new Set();
-	private _gids: Map<string, Map<string, Entry<T>>>; // gid -> hash -> entry
-	private _headsCache: HeadsCache<T> | undefined;
-	private _config: Log<T>;
-	private _onGidRemoved?: (gid: string[]) => Promise<void> | void;
-	constructor(id: Uint8Array) {
-		this._gids = new Map();
-		this._id = id;
-	}
-
-	async init(
-		log: Log<T>,
-		options: {
-			entries?: Entry<T>[];
-			onGidRemoved?: (gid: string[]) => Promise<void> | void;
-		} = {}
-	) {
-		this._config = log;
-		this._onGidRemoved = options.onGidRemoved;
-		await this.reset(options?.entries || []);
-		if (log.memory) {
-			this._headsCache = new HeadsCache(this);
-			return this._headsCache.init(await log.memory.sublevel("heads"));
-		}
-	}
-
-	async load(
-		options?: {
-			timeout?: number;
-			replicate?: boolean;
-			reload?: boolean;
-			ignoreMissing?: boolean;
-		} & CacheUpdateOptions
-	): Promise<Entry<T>[] | undefined> {
-		if (!this._headsCache || (this._headsCache.loaded && !options?.reload)) {
-			return;
-		}
-
-		// TODO make below into a promise that concurrenct caklls can wait on?
-		const heads = await this._headsCache?.load();
-		if (!heads) {
-			return;
-		}
-		const entries = await Promise.all(
-			heads.map(async (x) => {
-				const entry = await this._config.entryIndex.get(x, { load: true });
-				if (!entry) {
-					if (options?.ignoreMissing) {
-						logger.error("Failed to load entry from head with hash: " + x);
-						return;
-					} else {
-						throw new Error("Failed to load entry from head with hash: " + x);
-					}
-				}
-				await entry.getMeta(); // TODO types,decrypt gid
-				return entry;
-			})
-		);
-		const filtered = entries.filter((x): x is Entry<any> => !!x);
-		await this.reset(filtered);
-		return filtered;
-	}
-
-	get headsCache(): HeadsCache<T> | undefined {
-		return this._headsCache;
-	}
-
-	close() {
-		return this._headsCache?.close();
-	}
-
-	drop() {
-		return this._headsCache?.drop();
-	}
-
-	get id(): Uint8Array {
-		return this._id;
-	}
-
-	get index() {
-		return this._index;
-	}
-
-	get gids(): Map<string, Map<string, Entry<T>>> {
-		return this._gids;
-	}
-
-	get size() {
-		return this._index.size;
-	}
-
-	async reset(
-		entries: Entry<T>[],
-		options: CacheUpdateOptions = { cache: { reset: true, update: true } }
-	) {
-		this._index.clear();
-		const gidKeys = [...this._gids.keys()];
-
-		this._gids = new Map();
-		if (entries?.length > 0) {
-			await this.putAll(entries, options); // reset cache = true
-		}
-
-		if (gidKeys.length > 0) {
-			this._onGidRemoved?.(gidKeys);
-		}
-	}
-
-	has(cid: string) {
-		return this._index.has(cid);
-	}
-
-	async put(entry: Entry<T>, options?: CacheUpdateOptions) {
-		await this._putOne(entry);
-		if (!options?.cache || options?.cache?.update) {
-			await this._headsCache?.queue({ added: [entry] }, options?.cache?.reset);
-		}
-	}
-
-	async putAll(entries: Entry<T>[], options?: CacheUpdateOptions) {
-		await this._putAll(entries);
-		if (!options?.cache || options?.cache?.update) {
-			await this._headsCache?.queue({ added: entries }, options?.cache?.reset);
-		}
-	}
-
-	async resetHeadsCache() {
-		await this._headsCache?.queue(
-			{ added: [...this._index], removed: [] },
-			true
-		);
-	}
-	async updateHeadsCache(
-		change: {
-			added?: (Entry<T> | string)[];
-			removed?: (Entry<T> | string)[];
-		} = {},
-		reset?: boolean
-	) {
-		await this._headsCache?.queue(change, reset);
-	}
-
-	private async _putOne(entry: Entry<T>) {
-		if (!entry.hash) {
-			throw new Error("Missing hash");
-		}
-		if (this._index.has(entry.hash)) {
-			return;
-		}
-
-		this._index.add(entry.hash);
-		const map = this._gids.get(entry.meta.gid);
-		if (!map) {
-			const newMap = new Map();
-			this._gids.set(entry.meta.gid, newMap);
-			newMap.set(entry.hash, entry);
-		} else {
-			map.set(entry.hash, entry);
-		}
-
-		for (const next of entry.next) {
-			const indexedEntry = this._config.entryIndex.getShallow(next);
-			if (indexedEntry) {
-				await this.del(indexedEntry, { cache: { update: false } }); // we dont update cache here because the put will update the cache that is calling _putOne
-			}
-		}
-	}
-
-	private async _putAll(entries: Entry<T>[]) {
-		for (const entry of entries) {
-			await this._putOne(entry);
-		}
-	}
-
-	async del(
-		entry: { hash: string; meta: { gid: string } },
-		options?: CacheUpdateOptions
-	): Promise<boolean> {
-		const wasHead = this._index.delete(entry.hash);
-		if (!wasHead) {
-			return false;
-		}
-		let removedGids: Set<string> | undefined = undefined;
-		const map = this._gids.get(entry.meta.gid)!;
-		map.delete(entry.hash);
-		if (map.size <= 0) {
-			this._gids.delete(entry.meta.gid);
-			(removedGids || (removedGids = new Set<string>())).add(entry.meta.gid);
-		}
-
-		if (!entry.hash) {
-			throw new Error("Missing hash");
-		}
-
-		if (removedGids) {
-			await this._onGidRemoved?.([...removedGids]);
-		}
-
-		if (wasHead && !(options?.cache || options?.cache?.update)) {
-			await this._headsCache?.queue(
-				{ removed: [entry.hash] },
-				options?.cache?.reset
-			);
-		}
-
-		return wasHead;
-		//     this._headsCache = undefined; // TODO do smarter things here, only remove the element needed (?)
-	}
-}

package/src/types.ts

@@ -1,12 +0,0 @@
-import { field, vec } from "@dao-xyz/borsh";
-
-export class StringArray {
-	@field({ type: vec("string") })
-	arr: string[];
-
-	constructor(properties?: { arr: string[] }) {
-		if (properties) {
-			this.arr = properties.arr;
-		}
-	}
-}

package/src/values.ts

@@ -1,174 +0,0 @@
-import { Entry, ShallowEntry } from "./entry";
-import { ISortFunction } from "./log-sorting";
-import yallist from "yallist";
-import { EntryIndex } from "./entry-index";
-
-export type EntryNode = yallist.Node<string>;
-
-export class Values<T> {
-	/**
-	 * Keep track of sorted elements in descending sort order (i.e. newest elements)
-	 */
-	private _values: yallist<string>;
-	private _sortFn: ISortFunction;
-	private _byteLength: number;
-	private _entryIndex: EntryIndex<T>;
-
-	constructor(
-		entryIndex: EntryIndex<T>,
-		sortFn: ISortFunction,
-		entries: Entry<T>[] = []
-	) {
-		this._values = yallist.create(
-			entries
-				.slice()
-				.sort(sortFn)
-				.reverse()
-				.map((x) => {
-					if (!x.hash) throw new Error("Unexpected");
-					return x.hash; /* {
-						hash: x.hash,
-						byteLength: x._payload.byteLength,
-						meta: {
-							gids: x.gids,
-							gid: x.gid,
-							data: x.meta.data,
-						},
-					}; */
-				})
-		);
-		this._byteLength = 0;
-		entries.forEach((entry) => {
-			this._byteLength += entry._payload.byteLength;
-		});
-		this._sortFn = sortFn;
-		this._entryIndex = entryIndex;
-	}
-
-	toArray(): Promise<Entry<T>[]> {
-		return Promise.all(
-			this._values.toArrayReverse().map((x) => this._entryIndex.get(x))
-		).then((arr) => arr.filter((x) => !!x)) as Promise<Entry<T>[]>; // we do reverse because we assume the log is only meaningful if we read it from start to end
-	}
-
-	get head() {
-		return this._values.head;
-	}
-	get tail() {
-		return this._values.tail;
-	}
-	get length() {
-		return this._values.length;
-	}
-
-	get entryIndex(): EntryIndex<T> {
-		return this._entryIndex;
-	}
-	put(value: Entry<T>) {
-		return this._put(value);
-	}
-	_put(value: Entry<T>) {
-		// assume we want to insert at head (or somehere close)
-		let walker = this._values.head;
-		let last: EntryNode | undefined = undefined;
-		while (walker) {
-			const walkerValue = this._entryIndex.getShallow(walker.value);
-			if (!walkerValue) {
-				throw new Error("Missing walker value");
-			}
-			if (walkerValue.hash === value.hash) {
-				return; // already exist!
-			}
-
-			if (this._sortFn(walkerValue, value) < 0) {
-				break;
-			}
-			last = walker;
-			walker = walker.next;
-			continue;
-		}
-
-		this._byteLength += value._payload.byteLength;
-		if (!value.hash) {
-			throw new Error("Unexpected");
-		}
-
-		_insertAfter(this._values, last, value.hash);
-	}
-
-	async delete(value: Entry<T> | string) {
-		const hash = typeof value === "string" ? value : value.hash;
-		// Assume we want to delete at tail (or somwhere close)
-
-		let walker = this._values.tail;
-		while (walker) {
-			const walkerValue = this._entryIndex.getShallow(walker.value);
-
-			if (!walkerValue) {
-				throw new Error("Missing walker value");
-			}
-
-			if (walkerValue.hash === hash) {
-				this._values.removeNode(walker);
-				this._byteLength -= walkerValue.payloadByteLength;
-				return;
-			}
-			walker = walker.prev; // prev will be undefined if you do removeNode(walker)
-		}
-		throw new Error(
-			"Failed to delete, entry does not exist" +
-				" ??? " +
-				this.length +
-				" ??? " +
-				hash
-		);
-	}
-
-	deleteNode(node: EntryNode) {
-		this._values.removeNode(node);
-		this._byteLength -= this._entryIndex.getShallow(
-			node.value
-		)!.payloadByteLength;
-		return;
-	}
-
-	pop() {
-		const value = this._values.pop();
-		if (value) {
-			this._byteLength -= this._entryIndex.getShallow(value)!.payloadByteLength;
-		}
-		return value;
-	}
-
-	get byteLength() {
-		return this._byteLength;
-	}
-}
-
-function _insertAfter(
-	self: yallist<any>,
-	node: EntryNode | undefined,
-	value: string
-) {
-	const inserted = !node
-		? new yallist.Node(
-				value,
-				null as any,
-				self.head as EntryNode | undefined,
-				self
-			)
-		: new yallist.Node(value, node, node.next as EntryNode | undefined, self);
-
-	// is tail
-	if (inserted.next === null) {
-		self.tail = inserted;
-	}
-
-	// is head
-	if (inserted.prev === null) {
-		self.head = inserted;
-	}
-
-	self.length++;
-	return inserted;
-}