@peerbit/log 1.0.2

package/src/find-uniques.ts

@@ -0,0 +1,14 @@
+export const findUniques = <T>(value: T[], key?: string): T[] => {
+	// Create an index of the collection
+	// TODO fix types. This method is quite ugly, maybe lets remove it altogether
+	const uniques: { [key: string | number | symbol]: T } = {};
+	const get = (key: string | number | symbol) => uniques[key];
+	const addToIndex = (e: T) =>
+		(uniques[
+			key
+				? ((e as any)[key] as string | number | symbol)
+				: (e as string | number | symbol)
+		] = e);
+	value.forEach(addToIndex);
+	return Object.keys(uniques).map(get);
+};

package/src/heads-cache.ts

@@ -0,0 +1,400 @@
+import PQueue from "p-queue";
+import { v4 as uuid } from "uuid";
+import { Entry } from "./entry";
+import { SimpleLevel } from "@peerbit/lazy-level";
+import { variant, option, field, vec } from "@dao-xyz/borsh";
+import { serialize, deserialize } from "@dao-xyz/borsh";
+import { logger as loggerFn } from "@peerbit/logger";
+import path from "path-browserify";
+export const logger = loggerFn({ module: "heads-cache" });
+export class CachedValue {}
+/* export type AppendOptions<T> = {
+	signers?: ((data: Uint8Array) => Promise<SignatureWithKey>)[];
+	nexts?: Entry<T>[];
+	reciever?: EncryptionTemplateMaybeEncrypted;
+	type?: EntryType;
+};
+ */
+
+@variant(0)
+export class CachePath {
+	@field({ type: "string" })
+	path: string;
+
+	constructor(path: string) {
+		this.path = path;
+	}
+}
+
+@variant(0)
+export class UnsfinishedReplication {
+	@field({ type: vec("string") })
+	hashes: string[];
+
+	constructor(opts?: { hashes: string[] }) {
+		if (opts) {
+			this.hashes = opts.hashes;
+		}
+	}
+}
+
+@variant(0)
+export class HeadsCacheToSerialize {
+	@field({ type: vec("string") })
+	heads: string[];
+
+	@field({ type: option("string") })
+	last?: string;
+
+	@field({ type: "u64" })
+	counter: bigint;
+
+	constructor(heads: string[], counter: bigint, last?: string) {
+		this.heads = heads;
+		this.last = last;
+		this.counter = counter;
+	}
+}
+
+const updateHashes = async (
+	headCache: HeadsCache<any>,
+	headsPath: string,
+	lastCid: string | undefined,
+	lastCounter: bigint,
+	hashes: string[]
+): Promise<{ counter: bigint; newPath: string }> => {
+	const newHeadsPath = path.join(
+		headsPath,
+		String(headCache.headsPathCounter),
+		uuid()
+	);
+	const counter = lastCounter + BigInt(hashes.length);
+	await Promise.all([
+		headCache.cache?.put(
+			headsPath,
+			serialize(new CachePath(newHeadsPath.toString()))
+		),
+		headCache.cache?.put(
+			newHeadsPath,
+			serialize(new HeadsCacheToSerialize(hashes, counter, lastCid))
+		),
+	]);
+	return { counter, newPath: newHeadsPath };
+};
+
+interface HeadsIndex {
+	id: Uint8Array;
+	size: number;
+	has(cid: string): boolean;
+}
+export class HeadsCache<T> /* implements Initiable<T>  */ {
+	// An access controller that is note part of the store manifest, usefull for circular store -> access controller -> store structures
+	headsPath: string;
+	removedHeadsPath: string;
+	initialized: boolean;
+
+	private _headsPathCounter = 0;
+
+	private _lastHeadsPath?: string;
+	private _lastHeadsCount = 0n;
+
+	private _lastRemovedHeadsPath?: string;
+	private _lastRemovedHeadsCount = 0n;
+
+	private _cache?: SimpleLevel;
+	private _cacheWriteQueue?: PQueue<any, any>;
+
+	private _loaded = false;
+	private _index: HeadsIndex;
+
+	constructor(index: HeadsIndex) {
+		this._index = index;
+	}
+
+	get cache(): SimpleLevel | undefined {
+		return this._cache;
+	}
+
+	get headsPathCounter(): number {
+		return this._headsPathCounter;
+	}
+
+	async init(cache?: SimpleLevel): Promise<this> {
+		if (this.initialized) {
+			throw new Error("Already initialized");
+		}
+
+		this._cache = cache;
+
+		// Set the options (we will use the topic property after thiis)
+		await this._cache?.open();
+
+		this.headsPath = "heads";
+		this.removedHeadsPath = "heads_removed";
+
+		await this.loadLastHeadsPath();
+
+		// append and log-joins queue. Adding ops and joins to the queue
+		// makes sure they get processed sequentially to avoid race conditions
+		this._cacheWriteQueue = new PQueue({ concurrency: 1 });
+		/* if (this._options.onOpen) {
+			await this._options.onOpen();
+		} */
+		this.initialized = true;
+
+		return this;
+	}
+
+	get loaded(): boolean {
+		return this._loaded;
+	}
+
+	private async _updateCachedHeads(
+		change: {
+			added?: (Entry<T> | string)[];
+			removed?: (Entry<T> | string)[];
+		},
+		reset?: boolean
+	) {
+		if (typeof reset !== "boolean" && change.added) {
+			// Only reset all heads if loaded once, since we don't want too loose track of unloaded heads
+			if (this._loaded && this._index.size <= change.added.length) {
+				let addedIsAllHeads = true;
+				for (const entry of change.added) {
+					const hash = typeof entry === "string" ? entry : entry.hash;
+					if (!this._index.has(hash)) {
+						addedIsAllHeads = false;
+					}
+				}
+				reset = addedIsAllHeads;
+			} else {
+				// added size < head size, meaning we have not rewritten all heads
+				reset = false;
+			}
+		}
+
+		// If 'reset' then dont keep references to old heads caches, assume new cache will fully describe all heads
+
+		// TODO dont delete old before saving new
+		if (reset) {
+			await this.cache?.clear();
+			this._lastHeadsPath = undefined;
+			this._lastRemovedHeadsPath = undefined;
+			this._lastHeadsCount = 0n;
+			this._lastRemovedHeadsCount = 0n;
+			this._headsPathCounter += 1;
+		}
+
+		if (change.added && change.added.length > 0) {
+			const update = await updateHashes(
+				this,
+				this.headsPath,
+				this._lastHeadsPath,
+				this._lastHeadsCount,
+				change.added.map((x) => (typeof x === "string" ? x : x.hash))
+			);
+			this._lastHeadsPath = update.newPath;
+			this._lastHeadsCount = update.counter;
+		}
+
+		if (this._lastHeadsPath) {
+			// only add removed heads if we actually have added heads, else these are pointless
+			if (change.removed && change.removed.length > 0) {
+				const update = await updateHashes(
+					this,
+					this.removedHeadsPath,
+					this._lastRemovedHeadsPath,
+					this._lastRemovedHeadsCount,
+					change.removed.map((x) => (typeof x === "string" ? x : x.hash))
+				);
+				this._lastRemovedHeadsPath = update.newPath;
+				this._lastRemovedHeadsCount = update.counter;
+				if (
+					update.counter > 0n &&
+					2n * update.counter >= this._lastHeadsCount
+				) {
+					const resetToHeads = await this.getCachedHeads(
+						this._lastHeadsPath,
+						this._lastRemovedHeadsPath
+					);
+					await this._updateCachedHeads(
+						{ added: resetToHeads, removed: [] },
+						true
+					);
+				}
+			}
+		}
+	}
+
+	async idle(): Promise<void> {
+		// Wait for the operations queue to finish processing
+		// to make sure everything that all operations that have
+		// been queued will be written to disk
+		await this._cacheWriteQueue?.onIdle();
+		await this._cache?.idle?.();
+	}
+
+	async getCachedHeads(
+		lastHeadsPath: string | undefined = this._lastHeadsPath,
+		lastRemovedHeadsPath: string | undefined = this._lastRemovedHeadsPath
+	): Promise<string[]> {
+		if (!this._cache) {
+			return [];
+		}
+		const getHashes = async (
+			start: string | undefined,
+			filter?: Set<string>
+		) => {
+			const result: string[] = [];
+			let next = start;
+			while (next) {
+				const cache = await this._cache
+					?.get(next)
+					.then((bytes) => bytes && deserialize(bytes, HeadsCacheToSerialize));
+				next = cache?.last;
+				cache?.heads.forEach((head) => {
+					if (filter && filter.has(head)) {
+						return;
+					}
+
+					result.push(head);
+				});
+			}
+			return result;
+		};
+
+		const removedHeads = new Set(await getHashes(lastRemovedHeadsPath));
+		const heads = await getHashes(lastHeadsPath, removedHeads);
+		return heads; // Saved heads - removed heads
+	}
+
+	/* get logOptions(): LogOptions<T> {
+		return {
+			logId: this.id,
+			trim: this._options.trim && {
+				// I can trim if I am not a replicator of an entry
+
+				...this._options.trim,
+				filter: this.options.replicator && {
+					canTrim: async (gid) => !(await this.options.replicator!(gid)),
+					cacheId: this.options.replicatorsCacheId,
+				},
+			},
+		};
+	} */
+
+	get closed() {
+		return !this._cache || this._cache.status === "closed";
+	}
+
+	async close() {
+		await this.idle();
+		await this._cache?.close();
+		this._loaded = false;
+		this._lastHeadsPath = undefined;
+		this._lastRemovedHeadsPath = undefined;
+		this._lastRemovedHeadsCount = 0n;
+		this._lastHeadsCount = 0n;
+	}
+
+	/**
+	 * Drops a database and removes local data
+	 */
+	async drop() {
+		this.initialized = false;
+
+		if (!this._cache) {
+			return; // already dropped
+		}
+		if (this._cache.status !== "open") {
+			await this._cache.open();
+		}
+
+		await this._cache.del(this.headsPath);
+		await this._cache.del(this.removedHeadsPath);
+		await this.close();
+
+		delete this._cache;
+	}
+
+	private async _loadHeads(): Promise<string[]> {
+		if (!this.initialized) {
+			throw new Error("Store needs to be initialized before loaded");
+		}
+
+		if (this._cache!.status !== "open") {
+			await this._cache!.open();
+		}
+
+		await this.loadLastHeadsPath();
+		return this.getCachedHeads();
+	}
+
+	async load() {
+		if (!this.initialized) {
+			throw new Error("Needs to be initialized before loaded");
+		}
+
+		if (this._cache!.status !== "open") {
+			await this._cache!.open();
+		}
+
+		const heads = await this._loadHeads();
+		this._loaded = true;
+		return heads;
+	}
+
+	async loadLastHeadsPath() {
+		this._lastHeadsPath = await this._cache
+			?.get(this.headsPath)
+			.then((bytes) => bytes && deserialize(bytes, CachePath).path);
+		this._lastRemovedHeadsPath = await this._cache
+			?.get(this.removedHeadsPath)
+			.then((bytes) => bytes && deserialize(bytes, CachePath).path);
+		this._lastHeadsCount = this._lastHeadsPath
+			? await this.getCachedHeadsCount(this._lastHeadsPath)
+			: 0n;
+		this._lastRemovedHeadsCount = this._lastRemovedHeadsPath
+			? await this.getCachedHeadsCount(this._lastRemovedHeadsPath)
+			: 0n;
+	}
+
+	async getCachedHeadsCount(headPath?: string): Promise<bigint> {
+		if (!headPath) {
+			return 0n;
+		}
+		return (
+			(
+				await this._cache
+					?.get(headPath)
+					.then((bytes) => bytes && deserialize(bytes, HeadsCacheToSerialize))
+			)?.counter || 0n
+		);
+	}
+
+	async waitForHeads() {
+		if (this.closed) {
+			throw new Error("Store is closed");
+		}
+		if (!this._loaded) {
+			return this._cacheWriteQueue?.add(async () => {
+				if (this._loaded) {
+					return;
+				}
+				return this.load();
+			});
+		}
+	}
+
+	public queue(
+		changes: {
+			added?: (Entry<T> | string)[];
+			removed?: (Entry<T> | string)[];
+		},
+		reset?: boolean
+	) {
+		return this._cacheWriteQueue?.add(() =>
+			this._updateCachedHeads(changes, reset)
+		);
+	}
+}

package/src/heads.ts

@@ -0,0 +1,208 @@
+import { Entry } from "./entry.js";
+import { SimpleLevel } from "@peerbit/lazy-level";
+import { HeadsCache } from "./heads-cache.js";
+import { Blocks } from "@peerbit/blocks-interface";
+import { Keychain } from "@peerbit/crypto";
+import { Encoding } from "./encoding.js";
+
+export type CacheUpdateOptions = {
+	cache?: { update?: false; reset?: false } | { update: true; reset?: boolean };
+};
+
+interface Config {
+	storage: Blocks;
+	keychain?: Keychain;
+	memory?: SimpleLevel;
+	encoding: Encoding<any>;
+}
+export class HeadsIndex<T> {
+	private _id: Uint8Array;
+	private _index: Set<string> = new Set();
+	private _gids: Map<string, number>;
+	private _headsCache: HeadsCache<T> | undefined;
+	private _config: Config;
+	constructor(id: Uint8Array) {
+		this._gids = new Map();
+		this._id = id;
+	}
+
+	async init(config: Config, options: { entries?: Entry<T>[] } = {}) {
+		this._config = config;
+		await this.reset(options?.entries || []);
+		if (config.memory) {
+			this._headsCache = new HeadsCache(this);
+			return this._headsCache.init(await config.memory.sublevel("heads"));
+		}
+	}
+
+	async load(
+		options?: {
+			timeout?: number;
+			replicate?: boolean;
+			reload?: boolean;
+		} & CacheUpdateOptions
+	) {
+		if (!this._headsCache || (this._headsCache.loaded && !options?.reload)) {
+			return;
+		}
+
+		// TODO make below into a promise that concurrenct caklls can wait on?
+		const heads = await this._headsCache?.load();
+		if (!heads) {
+			return;
+		}
+		try {
+			const entries = await Promise.all(
+				heads.map(async (x) => {
+					const entry = await Entry.fromMultihash<T>(
+						this._config.storage,
+						x,
+						options
+					);
+					entry.init(this._config);
+					await entry.getGid(); // decrypt gid
+					return entry;
+				})
+			);
+			await this.reset(entries);
+			return entries;
+		} catch (error) {
+			const q = 123;
+			throw error;
+		}
+	}
+
+	get headsCache(): HeadsCache<T> | undefined {
+		return this._headsCache;
+	}
+
+	close() {
+		return this._headsCache?.close();
+	}
+
+	drop() {
+		return this._headsCache?.drop();
+	}
+
+	get id(): Uint8Array {
+		return this._id;
+	}
+
+	get index() {
+		return this._index;
+	}
+
+	get gids(): Map<string, number> {
+		return this._gids;
+	}
+
+	get size() {
+		return this._index.size;
+	}
+
+	async reset(
+		entries: Entry<T>[],
+		options: CacheUpdateOptions = { cache: { reset: true, update: true } }
+	) {
+		this._index.clear();
+		this._gids = new Map();
+		if (entries) {
+			await this.putAll(entries, options); // reset cache = true
+		}
+	}
+
+	has(cid: string) {
+		return this._index.has(cid);
+	}
+
+	async put(entry: Entry<T>, options?: CacheUpdateOptions) {
+		this._putOne(entry);
+		if (options?.cache?.update) {
+			await this._headsCache?.queue({ added: [entry] }, options.cache.reset);
+		}
+	}
+
+	async putAll(entries: Entry<T>[], options?: CacheUpdateOptions) {
+		this._putAll(entries);
+		if (options?.cache?.update) {
+			await this._headsCache?.queue({ added: entries }, options.cache.reset);
+		}
+	}
+
+	async resetHeadsCache() {
+		await this._headsCache?.queue(
+			{ added: [...this._index], removed: [] },
+			true
+		);
+	}
+	async updateHeadsCache(
+		change: {
+			added?: (Entry<T> | string)[];
+			removed?: (Entry<T> | string)[];
+		} = {},
+		reset?: boolean
+	) {
+		await this._headsCache?.queue(change, reset);
+	}
+
+	private _putOne(entry: Entry<T>) {
+		if (!entry.hash) {
+			throw new Error("Missing hash");
+		}
+		if (this._index.has(entry.hash)) {
+			return;
+		}
+
+		this._index.add(entry.hash);
+		if (!this._gids.has(entry.gid)) {
+			this._gids.set(entry.gid, 1);
+		} else {
+			this._gids.set(entry.gid, this._gids.get(entry.gid)! + 1);
+		}
+	}
+
+	private _putAll(entries: Entry<T>[]) {
+		for (const entry of entries) {
+			this._putOne(entry);
+		}
+	}
+
+	async del(
+		entry: { hash: string; gid: string },
+		options?: CacheUpdateOptions
+	): Promise<{
+		removed: boolean;
+		lastWithGid: boolean;
+	}> {
+		const wasHead = this._index.delete(entry.hash);
+		if (!wasHead) {
+			return {
+				lastWithGid: false,
+				removed: false,
+			};
+		}
+		const newValue = this._gids.get(entry.gid)! - 1;
+		const lastWithGid = newValue <= 0;
+		if (newValue <= 0) {
+			this._gids.delete(entry.gid);
+		} else {
+			this._gids.set(entry.gid, newValue);
+		}
+		if (!entry.hash) {
+			throw new Error("Missing hash");
+		}
+
+		if (wasHead && options?.cache?.update) {
+			await this._headsCache?.queue(
+				{ removed: [entry.hash] },
+				options.cache.reset
+			);
+		}
+
+		return {
+			removed: wasHead,
+			lastWithGid: lastWithGid,
+		};
+		//     this._headsCache = undefined; // TODO do smarter things here, only remove the element needed (?)
+	}
+}

package/src/hrtime.ts

@@ -0,0 +1,78 @@
+/* 
+
+MIT License
+
+Copyright (c) 2020 Vlad Tansky
+Copyright (c) 2022 dao.xyz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+const _perfomancePolyfill = () => {
+	// based on https://gist.github.com/paulirish/5438650 copyright Paul Irish 2015.
+	if ("performance" in window === false) {
+		(window.performance as any) = {};
+	}
+
+	Date.now =
+		Date.now ||
+		(() => {
+			// thanks IE8
+			return new Date().getTime();
+		});
+
+	if ("now" in window.performance === false) {
+		let nowOffset = Date.now();
+
+		if (performance.timing && performance.timing.navigationStart) {
+			nowOffset = performance.timing.navigationStart;
+		}
+
+		window.performance.now = () => Date.now() - nowOffset;
+	}
+};
+
+const _hrtime = (previousTimestamp?: [number, number]): [number, number] => {
+	_perfomancePolyfill();
+	const baseNow = Math.floor((Date.now() - performance.now()) * 1e-3);
+	const clocktime = performance.now() * 1e-3;
+	let seconds = Math.floor(clocktime) + baseNow;
+	let nanoseconds = Math.floor((clocktime % 1) * 1e9);
+
+	if (previousTimestamp) {
+		seconds = seconds - previousTimestamp[0];
+		nanoseconds = nanoseconds - previousTimestamp[1];
+		if (nanoseconds < 0) {
+			seconds--;
+			nanoseconds += 1e9;
+		}
+	}
+	return [seconds, nanoseconds];
+};
+const NS_PER_SEC = 1e9;
+_hrtime.bigint = (time?: [number, number]): bigint => {
+	const diff = _hrtime(time);
+	return BigInt(diff[0] * NS_PER_SEC + diff[1]);
+};
+
+export default typeof process === "undefined" ||
+typeof process.hrtime === "undefined"
+	? _hrtime
+	: process.hrtime;

package/src/index.ts

@@ -0,0 +1,11 @@
+export * from "./log.js";
+export * from "./log-sorting.js";
+export * from "./log-errors.js";
+export * from "./snapshot.js";
+export * from "./entry.js";
+export * from "./utils.js";
+export * from "./clock.js";
+export * from "./encoding.js";
+export * from "./trim.js";
+export * from "./change.js";
+export type { TrimToByteLengthOption, TrimToLengthOption } from "./trim.js";

package/src/is-defined.ts

@@ -0,0 +1 @@
+export const isDefined = (arg: any) => arg !== undefined && arg !== null;