@peerbit/document 9.4.2 → 9.4.3-fb47029

package/package.json

@@ -1,76 +1,77 @@
 {
-	"name": "@peerbit/document",
-	"version": "9.4.2",
-	"description": "Document store implementation",
-	"type": "module",
-	"sideEffects": false,
-	"types": "./dist/src/index.d.ts",
-	"typesVersions": {
-		"*": {
-			"*": [
-				"*",
-				"dist/*",
-				"dist/src/*",
-				"dist/src/*/index"
-			],
-			"src/*": [
-				"*",
-				"dist/*",
-				"dist/src/*",
-				"dist/src/*/index"
-			]
-		}
-	},
-	"files": [
-		"src",
-		"dist",
-		"!dist/e2e",
-		"!dist/test",
-		"!**/*.tsbuildinfo"
-	],
-	"exports": {
-		".": {
-			"types": "./dist/src/index.d.ts",
-			"import": "./dist/src/index.js"
-		}
-	},
-	"eslintConfig": {
-		"extends": "peerbit",
-		"parserOptions": {
-			"project": true,
-			"sourceType": "module"
-		},
-		"ignorePatterns": [
-			"!.aegir.js",
-			"test/ts-use",
-			"*.d.ts"
-		]
-	},
-	"publishConfig": {
-		"access": "public"
-	},
-	"scripts": {
-		"clean": "aegir clean",
-		"build": "aegir build --no-bundle",
-		"test": "aegir test --target node",
-		"lint": "aegir lint"
-	},
-	"author": "dao.xyz",
-	"license": "MIT",
-	"dependencies": {
-		"@dao-xyz/borsh": "^5.2.3",
-		"@peerbit/program": "5.2.7",
-		"@peerbit/rpc": "5.2.9",
-		"@peerbit/shared-log": "11.0.7",
-		"@peerbit/indexer-interface": "^2.0.9",
-		"@peerbit/indexer-simple": "^1.1.14",
-		"@peerbit/indexer-sqlite3": "^1.2.18",
-		"@peerbit/document-interface": "^2.0.28"
-	},
-	"devDependencies": {
-		"@peerbit/test-utils": "2.1.42",
-		"@peerbit/time": "2.1.0",
-		"@types/pidusage": "^2.0.5",
-		"pidusage": "^3.0.2"
-	}
+  "name": "@peerbit/document",
+  "version": "9.4.3-fb47029",
+  "description": "Document store implementation",
+  "type": "module",
+  "sideEffects": false,
+  "types": "./dist/src/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "*": [
+        "*",
+        "dist/*",
+        "dist/src/*",
+        "dist/src/*/index"
+      ],
+      "src/*": [
+        "*",
+        "dist/*",
+        "dist/src/*",
+        "dist/src/*/index"
+      ]
+    }
+  },
+  "files": [
+    "src",
+    "dist",
+    "!dist/e2e",
+    "!dist/test",
+    "!**/*.tsbuildinfo"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "import": "./dist/src/index.js"
+    }
+  },
+  "eslintConfig": {
+    "extends": "peerbit",
+    "parserOptions": {
+      "project": true,
+      "sourceType": "module"
+    },
+    "ignorePatterns": [
+      "!.aegir.js",
+      "test/ts-use",
+      "*.d.ts"
+    ]
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "clean": "aegir clean",
+    "build": "aegir build --no-bundle",
+    "test": "aegir test --target node",
+    "lint": "aegir lint"
+  },
+  "author": "dao.xyz",
+  "license": "MIT",
+  "dependencies": {
+    "@dao-xyz/borsh": "^5.2.3",
+    "@peerbit/program": "5.2.7-fb47029",
+    "@peerbit/rpc": "5.2.9-fb47029",
+    "@peerbit/shared-log": "11.0.8-fb47029",
+    "@peerbit/indexer-interface": "2.0.9-fb47029",
+    "@peerbit/indexer-simple": "1.1.14-fb47029",
+    "@peerbit/indexer-sqlite3": "1.2.18-fb47029",
+    "@peerbit/document-interface": "2.0.28-fb47029",
+    "@peerbit/indexer-cache": "0.0.1-fb47029"
+  },
+  "devDependencies": {
+    "@peerbit/test-utils": "2.1.42-fb47029",
+    "@peerbit/time": "2.1.0-fb47029",
+    "@types/pidusage": "^2.0.5",
+    "pidusage": "^3.0.2"
+  }
 }

package/src/most-common-query-predictor.ts

@@ -0,0 +1,161 @@
+import { type AbstractType, deserialize, serialize } from "@dao-xyz/borsh";
+import { type PublicSignKey, randomBytes, toBase64 } from "@peerbit/crypto";
+import type * as types from "@peerbit/document-interface";
+
+/* ───────────────────── helpers ───────────────────── */
+
+const nullifyQuery = (
+	query: types.SearchRequest | types.SearchRequestIndexed,
+) => {
+	const cloned = deserialize(serialize(query), query.constructor) as
+		| types.SearchRequest
+		| types.SearchRequestIndexed;
+	cloned.id = new Uint8Array(32);
+	return cloned;
+};
+
+export const idAgnosticQueryKey = (
+	query: types.SearchRequest | types.SearchRequestIndexed,
+) => toBase64(serialize(nullifyQuery(query)));
+
+/* ───────────────────── predictor ───────────────────── */
+
+export interface QueryPredictor {
+	onRequest: (
+		request: types.SearchRequest | types.SearchRequestIndexed,
+		ctx: { from: PublicSignKey },
+	) => { ignore: boolean };
+
+	predictedQuery: (
+		from: PublicSignKey,
+	) => types.SearchRequest | types.SearchRequestIndexed | undefined;
+}
+
+interface QueryStats {
+	count: number;
+	lastSeen: number;
+	queryBytes: Uint8Array;
+	queryClazz: AbstractType<types.SearchRequest | types.SearchRequestIndexed>;
+}
+
+/**
+ * Learns the most common recent queries and predicts the most frequent one.
+ * If we just pre-empted a peer with some query, the *first* matching request
+ * that arrives from that peer within `ignoreWindow` ms is ignored.
+ */
+export default class MostCommonQueryPredictor implements QueryPredictor {
+	private readonly queries = new Map<string, QueryStats>();
+
+	/**
+	 * predicted:
+	 * requestKey  →  Map<peerHash, timestamp>
+	 */
+	private readonly predicted = new Map<string, Map<string, number>>();
+
+	constructor(
+		private readonly threshold: number,
+		private readonly ttl = 10 * 60 * 1000, // 10 min
+		private readonly ignoreWindow = 5_000, // 5 s
+	) {}
+
+	/* ───────── housekeeping ───────── */
+	private cleanupQueries(now: number) {
+		for (const [key, stats] of this.queries) {
+			if (now - stats.lastSeen > this.ttl) {
+				this.queries.delete(key);
+			}
+		}
+	}
+
+	private cleanupPredictions(now: number) {
+		for (const [key, peerMap] of this.predicted) {
+			for (const [peer, ts] of peerMap) {
+				if (now - ts > this.ignoreWindow) {
+					peerMap.delete(peer);
+				}
+			}
+			if (peerMap.size === 0) {
+				this.predicted.delete(key);
+			}
+		}
+	}
+
+	/* ───────── public API ───────── */
+
+	onRequest(
+		request: types.SearchRequest | types.SearchRequestIndexed,
+		{ from }: { from: PublicSignKey },
+	): { ignore: boolean } {
+		const now = Date.now();
+		const peerHash = from.hashcode();
+		const key = idAgnosticQueryKey(request);
+
+		/* — 1. Ignore if this (key, peer) pair was just predicted — */
+		const peerMap = this.predicted.get(key);
+		const ts = peerMap?.get(peerHash);
+		let ignore = false;
+		if (ts !== undefined && now - ts <= this.ignoreWindow) {
+			peerMap!.delete(peerHash); // one-shot
+			if (peerMap!.size === 0) {
+				this.predicted.delete(key);
+			}
+			ignore = true;
+		}
+
+		/* — 2. Learn from the request — */
+		const stats = this.queries.get(key);
+		if (stats) {
+			stats.count += 1;
+			stats.lastSeen = now;
+		} else {
+			this.queries.set(key, {
+				queryBytes: serialize(request),
+				queryClazz: request.constructor,
+				count: 1,
+				lastSeen: now,
+			});
+		}
+
+		/* — 3. Maintenance — */
+		this.cleanupQueries(now);
+		this.cleanupPredictions(now);
+
+		return { ignore };
+	}
+
+	predictedQuery(
+		from: PublicSignKey,
+	): types.SearchRequest | types.SearchRequestIndexed | undefined {
+		const now = Date.now();
+		this.cleanupQueries(now);
+		this.cleanupPredictions(now);
+
+		/* pick the most frequent query meeting the threshold */
+		let winnerKey: string | undefined;
+		let winnerCount = 0;
+		for (const [key, { count }] of this.queries) {
+			if (count > winnerCount) {
+				winnerKey = key;
+				winnerCount = count;
+			}
+		}
+		if (!winnerKey || winnerCount < this.threshold) return undefined;
+
+		const winner = this.queries.get(winnerKey)!;
+		const cloned = deserialize(winner.queryBytes, winner.queryClazz) as
+			| types.SearchRequest
+			| types.SearchRequestIndexed;
+		cloned.id = randomBytes(32);
+
+		/* remember that we pre-empted `from` with this query */
+		const peerHash = from.hashcode();
+		let peerMap = this.predicted.get(winnerKey);
+		if (!peerMap) {
+			peerMap = new Map<string, number>();
+			this.predicted.set(winnerKey, peerMap);
+		}
+		peerMap.set(peerHash, now);
+
+		return cloned;
+	}
+}

package/src/operation.ts

@@ -26,15 +26,15 @@ export class PutWithKeyOperation extends Operation {
 // @deprecated
 /* @variant(1)
 export class PutAllOperation<T> extends Operation<T> {
-    @field({ type: vec(PutOperation) })
-    docs: PutOperation<T>[];
-
-    constructor(props?: { docs: PutOperation<T>[] }) {
-        super();
-        if (props) {
-            this.docs = props.docs;
-        }
-    }
+	@field({ type: vec(PutOperation) })
+	docs: PutOperation<T>[];
+
+	constructor(props?: { docs: PutOperation<T>[] }) {
+		super();
+		if (props) {
+			this.docs = props.docs;
+		}
+	}
 }
  */
 

package/src/prefetch.ts

@@ -0,0 +1,80 @@
+import { TypedEventEmitter } from "@libp2p/interface";
+import { Cache } from "@peerbit/cache";
+import type * as types from "@peerbit/document-interface";
+import type { RPCResponse } from "@peerbit/rpc/dist/src";
+import { idAgnosticQueryKey } from "./most-common-query-predictor";
+
+// --- typed helper ---------------------------------------------------------
+type AddEvent = {
+	consumable: RPCResponse<types.PredictedSearchRequest<any>>;
+};
+
+const prefetchKey = (
+	request: types.SearchRequest | types.SearchRequestIndexed,
+	keyHash: string,
+) => `${idAgnosticQueryKey(request)} - ${keyHash}`;
+
+// --------------------------------------------------------------------------
+export class Prefetch extends TypedEventEmitter<{
+	add: CustomEvent<AddEvent>;
+}> {
+	constructor(
+		private prefetch: Cache<
+			RPCResponse<types.PredictedSearchRequest<any>>
+		> = new Cache({
+			max: 100,
+			ttl: 1e4,
+		}),
+		private searchIdTranslationMap: Map<
+			string,
+			Map<string, Uint8Array>
+		> = new Map(),
+	) {
+		super();
+	}
+
+	/** Store the prediction **and** notify listeners */
+	public add(
+		request: RPCResponse<types.PredictedSearchRequest<any>>,
+		keyHash: string,
+	): void {
+		const key = prefetchKey(request.response.request, keyHash);
+		this.prefetch.add(key, request);
+		this.dispatchEvent(
+			new CustomEvent("add", { detail: { consumable: request } }),
+		);
+	}
+
+	public consume(
+		request: types.SearchRequest | types.SearchRequestIndexed,
+		keyHash: string,
+	): RPCResponse<types.PredictedSearchRequest<any>> | undefined {
+		const key = prefetchKey(request, keyHash);
+		const pre = this.prefetch.get(key);
+		if (!pre) return;
+
+		this.prefetch.del(key);
+
+		let peerMap = this.searchIdTranslationMap.get(request.idString);
+		if (!peerMap) {
+			peerMap = new Map();
+			this.searchIdTranslationMap.set(request.idString, peerMap);
+		}
+		peerMap.set(keyHash, pre.response.request.id);
+		return pre;
+	}
+
+	clear(request: types.SearchRequest | types.SearchRequestIndexed) {
+		this.searchIdTranslationMap.delete(request.idString);
+	}
+
+	getTranslationMap(
+		request: types.SearchRequest | types.SearchRequestIndexed,
+	): Map<string, Uint8Array> | undefined {
+		return this.searchIdTranslationMap.get(request.idString);
+	}
+
+	get size() {
+		return this.prefetch.size;
+	}
+}

package/src/program.ts

@@ -7,6 +7,7 @@ import {
 } from "@dao-xyz/borsh";
 import { AccessError } from "@peerbit/crypto";
 import { ResultIndexedValue } from "@peerbit/document-interface";
+import type { QueryCacheOptions } from "@peerbit/indexer-cache";
 import * as indexerTypes from "@peerbit/indexer-interface";
 import {
 	type Change,
@@ -39,6 +40,7 @@ import {
 	type CanRead,
 	type CanSearch,
 	DocumentIndex,
+	type PrefetchOptions,
 	type TransformOptions,
 	type WithContext,
 	coerceWithContext,
@@ -95,7 +97,11 @@ export type SetupOptions<
 		canSearch?: CanSearch;
 		canRead?: CanRead<I>;
 		idProperty?: string | string[];
-		cacheSize?: number;
+		cache?: {
+			resolver?: number;
+			query?: QueryCacheOptions;
+		};
+		prefetch?: boolean | Partial<PrefetchOptions>;
 	} & TransformOptions<T, I>;
 	log?: {
 		trim?: TrimOptions;
@@ -192,7 +198,7 @@ export class Documents<
 			transform: options.index,
 			indexBy: idProperty,
 			compatibility: options.compatibility,
-			cacheSize: options?.index?.cacheSize,
+			cache: options?.index?.cache,
 			replicate: async (query, results) => {
 				// here we arrive for all the results we want to persist.
 
@@ -210,6 +216,7 @@ export class Documents<
 			},
 			dbType: this.constructor,
 			maybeOpen: this.maybeSubprogramOpen.bind(this),
+			prefetch: options.index?.prefetch,
 		});
 
 		// document v6 and below need log compatibility of v8 or below

package/src/resumable-iterator.ts

@@ -62,7 +62,15 @@ export class ResumableIterators<T extends Record<string, any>> {
 		this.queues.del(id);
 	}
 
-	getPending(id: string) {
-		return this.queues.get(id)?.iterator.pending();
+	async getPending(id: string) {
+		let iterator = this.queues.get(id);
+		if (!iterator) {
+			return undefined;
+		}
+		const pending = await iterator.iterator.pending();
+		if (pending === 0 && iterator.iterator.done()) {
+			this.clear(id);
+		}
+		return pending;
 	}
 }