@peerbit/document 15.1.6 → 15.1.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/document",
-  "version": "15.1.6",
+  "version": "15.1.7",
   "description": "Document store implementation",
   "type": "module",
   "sideEffects": false,
@@ -61,20 +61,21 @@
     "p-defer": "^4.0.0",
     "uint8arrays": "^5.1.0",
     "@peerbit/cache": "3.1.1",
-    "@peerbit/document-interface": "3.2.79",
     "@peerbit/crypto": "3.1.6",
+    "@peerbit/document-interface": "3.2.79",
     "@peerbit/indexer-cache": "0.3.0",
-    "@peerbit/indexer-interface": "3.1.0",
+    "@peerbit/diagnostics": "0.1.0",
     "@peerbit/indexer-simple": "1.3.0",
+    "@peerbit/log": "6.2.35",
     "@peerbit/indexer-sqlite3": "3.0.21",
     "@peerbit/logger": "2.0.2",
-    "@peerbit/log": "6.2.35",
-    "@peerbit/program": "6.0.63",
-    "@peerbit/rpc": "6.1.31",
-    "@peerbit/pubsub": "5.4.8",
-    "@peerbit/shared-log": "16.0.35",
+    "@peerbit/program": "6.0.64",
+    "@peerbit/pubsub": "5.4.9",
+    "@peerbit/indexer-interface": "3.1.0",
     "@peerbit/stream-interface": "6.0.16",
-    "@peerbit/time": "3.0.1"
+    "@peerbit/time": "3.0.1",
+    "@peerbit/shared-log": "16.0.36",
+    "@peerbit/rpc": "6.2.0"
   },
   "optionalDependencies": {
     "@peerbit/document-rust": "0.1.1"
@@ -85,10 +86,10 @@
     "@types/pidusage": "^2.0.5",
     "pidusage": "^4.0.1",
     "uuid": "^11.1.1",
-    "@peerbit/log-rust": "1.1.5",
     "@peerbit/native-backbone": "0.2.17",
-    "@peerbit/test-utils": "3.1.44",
-    "peerbit": "5.4.5"
+    "@peerbit/test-utils": "3.1.45",
+    "@peerbit/log-rust": "1.1.5",
+    "peerbit": "5.4.6"
   },
   "repository": {
     "type": "git",

package/src/program.ts

@@ -2622,6 +2622,7 @@ export class Documents<
 			prefetch: options.index?.prefetch,
 			includeIndexed: options.index?.includeIndexed,
 			immutable: this.immutable,
+			profile: options.sync?.profile,
 		});
 		this._documentInternalChangeListenerCount = Math.max(
 			0,

package/src/search.ts

@@ -14,6 +14,10 @@ import {
 	PublicSignKey,
 	sha256Base64Sync,
 } from "@peerbit/crypto";
+import {
+	type DiagnosticSink,
+	createDiagnosticTrace,
+} from "@peerbit/diagnostics";
 import * as types from "@peerbit/document-interface";
 import { CachedIndex, type QueryCacheOptions } from "@peerbit/indexer-cache";
 import * as indexerTypes from "@peerbit/indexer-interface";
@@ -79,6 +83,11 @@ import {
 } from "./transform.js";
 
 const WARNING_WHEN_ITERATING_FOR_MORE_THAN = 1e5;
+const QUERY_PROFILE_TARGET_LIMIT = 16;
+type QueryProfile = {
+	trace: NonNullable<ReturnType<typeof createDiagnosticTrace>>;
+	missingGroups: number;
+};
 
 const logger = loggerFn("peerbit:program:document:search");
 const warn = logger.newScope("warn");
@@ -838,6 +847,8 @@ export type OpenOptions<
 	prefetch?: boolean | Partial<PrefetchOptions>;
 	includeIndexed?: boolean; // if true, indexed representations will always be included in the search results
 	immutable?: boolean; // conflict rule of the owning store: oldest version wins when true (mirrors Documents "immutable")
+	/** Advisory query timing; no effect on coverage, authorization or deadlines. */
+	profile?: DiagnosticSink;
 };
 
 type IndexableClass<I> = new (
@@ -1089,6 +1100,7 @@ export class DocumentIndex<
 	) => Promise<void>;
 
 	private _log: SharedLog<Operation, D, any>;
+	private _queryProfile?: DiagnosticSink;
 
 	private _resolverProgramCache?: Map<string | number | bigint, T>;
 	private _resolverCache?: Cache<T>;
@@ -1534,6 +1546,7 @@ export class DocumentIndex<
 	}
 	async open(properties: OpenOptions<T, I, D>) {
 		this._log = properties.log;
+		this._queryProfile = properties.profile;
 		// Allow reopening with partial options (tests override the index transform)
 		const previousEvents = this.documentEvents;
 		this.documentEvents =
@@ -2305,6 +2318,7 @@ export class DocumentIndex<
 	async close(from?: Program): Promise<boolean> {
 		const closed = await super.close(from);
 		if (closed) {
+			this._queryProfile = undefined;
 			if (this._joinListener) {
 				this._query.events.removeEventListener("join", this._joinListener);
 			}
@@ -2339,6 +2353,7 @@ export class DocumentIndex<
 	async drop(from?: Program): Promise<boolean> {
 		const dropped = await super.drop(from);
 		if (dropped) {
+			this._queryProfile = undefined;
 			this.documentEvents?.removeEventListener(
 				"change",
 				this.handleDocumentChange,
@@ -4210,7 +4225,55 @@ export class DocumentIndex<
 	 * @param options
 	 * @returns
 	 */
-	private async queryCommence<
+	private queryCommence<
+		R extends
+			| types.SearchRequest
+			| types.SearchRequestIndexed
+			| types.IterationRequest,
+		RT extends types.Result = types.ResultTypeFromRequest<R, T, I>,
+	>(
+		queryRequest: R,
+		options?: QueryDetailedOptions<T, I, D, boolean | undefined>,
+		fetchFirstForRemote?: Set<string>,
+	): Promise<types.Results<RT>[]> {
+		const trace = createDiagnosticTrace(
+			this._queryProfile,
+			"document",
+			"query",
+		);
+		if (!trace) {
+			return this.queryCommenceWithProfile<R, RT>(
+				queryRequest,
+				options,
+				fetchFirstForRemote,
+			);
+		}
+		const profile: QueryProfile = { trace, missingGroups: 0 };
+		let outcome = "rejected";
+		trace.emit({
+			name: "documents.query.start",
+			details: { immutable: this.immutable },
+		});
+		return (async () => {
+			try {
+				const results = await this.queryCommenceWithProfile<R, RT>(
+					queryRequest,
+					options,
+					fetchFirstForRemote,
+					profile,
+				);
+				outcome = "fulfilled";
+				return results;
+			} finally {
+				trace.finish({
+					name: "documents.query.settle",
+					details: { outcome, missingGroups: profile.missingGroups },
+				});
+			}
+		})();
+	}
+
+	private async queryCommenceWithProfile<
 		R extends
 			| types.SearchRequest
 			| types.SearchRequestIndexed
@@ -4220,7 +4283,9 @@ export class DocumentIndex<
 		queryRequest: R,
 		options?: QueryDetailedOptions<T, I, D, boolean | undefined>,
 		fetchFirstForRemote?: Set<string>,
+		profile?: QueryProfile,
 	): Promise<types.Results<RT>[]> {
+		const trace = profile?.trace;
 		const local = typeof options?.local === "boolean" ? options?.local : true;
 		let remote:
 			| RemoteQueryOptions<
@@ -4259,11 +4324,22 @@ export class DocumentIndex<
 			[];
 
 		if (local) {
+			const startedAt = trace?.now();
+			trace?.emit({
+				name: "documents.query.local",
+				details: { edge: "start" },
+			});
 			const results = await this.processQuery(
 				queryRequest,
 				this.node.identity.publicKey,
 				true,
 			);
+			trace?.emit({
+				name: "documents.query.local",
+				durationMs: trace.now() - startedAt!,
+				entries: results.results.length,
+				details: { edge: "end" },
+			});
 			if (results.results.length > 0) {
 				options?.onResponse &&
 					(await options.onResponse(results, this.node.identity.publicKey));
@@ -4284,6 +4360,11 @@ export class DocumentIndex<
 				(!("args" in coverProps) || (coverProps as any).args == null);
 			const remoteWasExplicit = options?.remote != null;
 
+			const coverStartedAt = trace?.now();
+			trace?.emit({
+				name: "documents.query.cover",
+				details: { edge: "start" },
+			});
 			let replicatorGroups = options?.remote?.from
 				? options?.remote?.from
 				: await this._log.getCover(coverProps, {
@@ -4292,6 +4373,16 @@ export class DocumentIndex<
 						reachableOnly: !!remote.wait, // when we want to merge joining we can ignore pending to be online peers and instead consider them once they become online
 						signal: options?.signal,
 					});
+			trace?.emit({
+				name: "documents.query.cover",
+				durationMs: trace.now() - coverStartedAt!,
+				targets: replicatorGroups.length,
+				details: {
+					edge: "end",
+					explicit: !!options?.remote?.from,
+					reachableOnly: !!remote.wait,
+				},
+			});
 
 			// Cold start: cover can be temporarily self-only or empty while
 			// replication metadata converges. For explicit bounded remote searches,
@@ -4365,6 +4456,11 @@ export class DocumentIndex<
 						from?: PublicSignKey;
 					}[],
 				) => {
+					const startedAt = trace?.now();
+					trace?.emit({
+						name: "documents.query.introduce",
+						details: { edge: "start" },
+					});
 					const resultInitialized = await introduceEntries(
 						queryRequest,
 						results,
@@ -4373,6 +4469,12 @@ export class DocumentIndex<
 						this._sync,
 						options,
 					);
+					trace?.emit({
+						name: "documents.query.introduce",
+						durationMs: trace.now() - startedAt!,
+						count: resultInitialized.length,
+						details: { edge: "end" },
+					});
 					for (const r of resultInitialized) {
 						resolved.push(r.response);
 					}
@@ -4425,9 +4527,54 @@ export class DocumentIndex<
 					.map((x) => [x]);
 
 				options?.onRemoteTargets?.(selectedRemoteHashes);
+				if (trace) {
+					trace.emit({
+						name: "documents.query.targets",
+						targets: selectedRemoteHashes.length,
+						details: {
+							requestTargets: groupHashes.length,
+							sampledTargets: Math.min(
+								QUERY_PROFILE_TARGET_LIMIT,
+								selectedRemoteHashes.length,
+							),
+						},
+					});
+					for (
+						let i = 0;
+						i <
+						Math.min(QUERY_PROFILE_TARGET_LIMIT, selectedRemoteHashes.length);
+						i++
+					) {
+						const hash = selectedRemoteHashes[i];
+						// Presence is only local direct-peer evidence, not reachability,
+						// subscriber readiness, authority or receipt eligibility.
+						const peers = (
+							this.node.services.pubsub as { peers?: Map<string, unknown> }
+						).peers;
+						trace.emit({
+							name: "documents.query.target",
+							peer: hash,
+							details: {
+								index: i,
+								directPeerPresent:
+									peers instanceof Map ? peers.has(hash) : undefined,
+							},
+						});
+					}
+				}
 				extraPromises && (await Promise.all(extraPromises));
 				let tearDown: (() => void) | undefined = undefined;
 				const search = this;
+				const requestOptions = trace
+					? {
+							...remote,
+							profile: (event: Parameters<DiagnosticSink>[0]) =>
+								trace.emit({
+									...event,
+									details: { ...event.details, requestTraceId: event.traceId },
+								}),
+						}
+					: remote;
 
 				try {
 					groupHashes.length > 0 &&
@@ -4438,7 +4585,7 @@ export class DocumentIndex<
 							responseHandler,
 							search._prefetch?.accumulator
 								? {
-										...remote,
+										...requestOptions,
 										responseInterceptor(fn) {
 											const listener = (evt: {
 												detail: {
@@ -4490,10 +4637,18 @@ export class DocumentIndex<
 											};
 										},
 									}
-								: remote,
+								: requestOptions,
 						));
 				} catch (error) {
 					if (error instanceof MissingResponsesError) {
+						if (profile) {
+							profile.missingGroups = error.missingGroups.length;
+							trace!.emit({
+								name: "documents.query.missing",
+								count: error.missingGroups.length,
+								details: { tolerated: !remote.throwOnMissing },
+							});
+						}
 						warn("Did not reciveve responses from all shard");
 						if (options?.onMissingResponses) {
 							await options.onMissingResponses(error);