@peerbit/document 11.1.1 → 11.2.1

package/package.json

@@ -1,93 +1,93 @@
 {
-  "name": "@peerbit/document",
-  "version": "11.1.1",
-  "description": "Document store implementation",
-  "type": "module",
-  "sideEffects": false,
-  "types": "./dist/src/index.d.ts",
-  "typesVersions": {
-    "*": {
-      "*": [
-        "*",
-        "dist/*",
-        "dist/src/*",
-        "dist/src/*/index"
-      ],
-      "src/*": [
-        "*",
-        "dist/*",
-        "dist/src/*",
-        "dist/src/*/index"
-      ]
-    }
-  },
-  "files": [
-    "src",
-    "dist",
-    "!dist/e2e",
-    "!dist/test",
-    "!**/*.tsbuildinfo"
-  ],
-  "exports": {
-    ".": {
-      "types": "./dist/src/index.d.ts",
-      "import": "./dist/src/index.js"
-    }
-  },
-  "eslintConfig": {
-    "extends": "peerbit",
-    "parserOptions": {
-      "project": true,
-      "sourceType": "module"
-    },
-    "ignorePatterns": [
-      "!.aegir.js",
-      "test/ts-use",
-      "*.d.ts"
-    ]
-  },
-  "publishConfig": {
-    "access": "public"
-  },
-  "author": "dao.xyz",
-  "license": "MIT",
-  "dependencies": {
-    "@dao-xyz/borsh": "^6.0.0",
-    "@chainsafe/libp2p-yamux": "^8.0.0",
-    "@libp2p/interface": "^3.1.0",
-    "@libp2p/tcp": "^11.0.2",
-    "@multiformats/multiaddr": "^13.0.1",
-    "p-defer": "^4.0.0",
-    "uint8arrays": "^5.1.0",
-    "@peerbit/cache": "2.2.0",
-    "@peerbit/log": "4.2.5",
-    "@peerbit/logger": "2.0.0",
-    "@peerbit/pubsub": "4.1.1",
-    "@peerbit/crypto": "2.4.0",
-    "@peerbit/program": "5.4.3",
-    "@peerbit/shared-log": "11.6.5",
-    "@peerbit/rpc": "5.4.5",
-    "@peerbit/indexer-interface": "2.1.0",
-    "@peerbit/indexer-simple": "1.2.0",
-    "@peerbit/indexer-sqlite3": "1.3.0",
-    "@peerbit/document-interface": "3.1.5",
-    "@peerbit/indexer-cache": "0.2.0",
-    "@peerbit/stream-interface": "5.3.0"
-  },
-  "devDependencies": {
-    "@types/pidusage": "^2.0.5",
-    "pidusage": "^4.0.1",
-    "uuid": "^10.0.0",
-    "@libp2p/websockets": "^10.1.0",
-    "@chainsafe/libp2p-noise": "^17.0.0",
-    "@peerbit/test-utils": "2.3.5",
-    "peerbit": "4.4.5",
-    "@peerbit/time": "2.3.0"
-  },
-  "scripts": {
-    "clean": "aegir clean",
-    "build": "aegir build --no-bundle",
-    "test": "aegir test --target node",
-    "lint": "aegir lint"
-  }
-}
+	"name": "@peerbit/document",
+	"version": "11.2.1",
+	"description": "Document store implementation",
+	"type": "module",
+	"sideEffects": false,
+	"types": "./dist/src/index.d.ts",
+	"typesVersions": {
+		"*": {
+			"*": [
+				"*",
+				"dist/*",
+				"dist/src/*",
+				"dist/src/*/index"
+			],
+			"src/*": [
+				"*",
+				"dist/*",
+				"dist/src/*",
+				"dist/src/*/index"
+			]
+		}
+	},
+	"files": [
+		"src",
+		"dist",
+		"!dist/e2e",
+		"!dist/test",
+		"!**/*.tsbuildinfo"
+	],
+	"exports": {
+		".": {
+			"types": "./dist/src/index.d.ts",
+			"import": "./dist/src/index.js"
+		}
+	},
+	"eslintConfig": {
+		"extends": "peerbit",
+		"parserOptions": {
+			"project": true,
+			"sourceType": "module"
+		},
+		"ignorePatterns": [
+			"!.aegir.js",
+			"test/ts-use",
+			"*.d.ts"
+		]
+	},
+	"publishConfig": {
+		"access": "public"
+	},
+	"scripts": {
+		"clean": "aegir clean",
+		"build": "aegir build --no-bundle",
+		"test": "aegir test --target node",
+		"lint": "aegir lint"
+	},
+	"author": "dao.xyz",
+	"license": "MIT",
+	"dependencies": {
+		"@dao-xyz/borsh": "^6.0.0",
+		"@chainsafe/libp2p-yamux": "^8.0.0",
+		"@libp2p/interface": "^3.1.0",
+		"@libp2p/tcp": "^11.0.2",
+		"@multiformats/multiaddr": "^13.0.1",
+		"@peerbit/cache": "workspace:*",
+		"@peerbit/crypto": "workspace:*",
+		"@peerbit/logger": "workspace:*",
+		"@peerbit/log": "workspace:*",
+		"@peerbit/pubsub": "workspace:*",
+		"@peerbit/program": "workspace:*",
+		"@peerbit/rpc": "workspace:*",
+		"@peerbit/shared-log": "workspace:*",
+		"@peerbit/indexer-interface": "workspace:*",
+		"@peerbit/indexer-simple": "workspace:*",
+		"@peerbit/indexer-sqlite3": "workspace:*",
+		"@peerbit/document-interface": "workspace:*",
+		"@peerbit/indexer-cache": "workspace:*",
+		"@peerbit/stream-interface": "workspace:*",
+		"p-defer": "^4.0.0",
+		"uint8arrays": "^5.1.0"
+	},
+	"devDependencies": {
+		"@peerbit/test-utils": "workspace:*",
+		"@peerbit/time": "workspace:*",
+		"peerbit": "workspace:*",
+		"@types/pidusage": "^2.0.5",
+		"pidusage": "^4.0.1",
+		"uuid": "^10.0.0",
+		"@libp2p/websockets": "^10.1.0",
+		"@chainsafe/libp2p-noise": "^17.0.0"
+	}
+}

package/src/search.ts

@@ -142,12 +142,32 @@ export type JoiningOnMissedResults = (evt: {
 	peer: PublicSignKey;
 }) => void | Promise<void>;
 
-export type LateResultsEvent = {
+export type OutOfOrderMode = "drop" | "queue";
+
+export type LateResultsItem = {
+	indexed: WithContext<any>;
+	context: types.Context;
+	from: PublicSignKey;
+	value?: any;
+};
+
+export type LateResultsEvent<
+	M extends OutOfOrderMode = "drop",
+	Item = LateResultsItem,
+> = {
 	/** Count of items that should have appeared earlier than the current frontier */
 	amount: number;
 
 	/** If attributable, the peer that produced the late items */
 	peer?: PublicSignKey;
+} & (M extends "queue" ? { items: Item[] } : { items?: undefined });
+
+export type LateResultsHelpers<
+	M extends OutOfOrderMode = "drop",
+	Item = LateResultsItem,
+> = {
+	/** Collect concrete late items if available for the chosen mode */
+	collect: () => Promise<M extends "queue" ? Item[] : Item[] | undefined>;
 };
 
 export type WaitBehavior =
@@ -197,7 +217,21 @@ export type QueryOptions<T, I, D, Resolve extends boolean | undefined> = {
 	resolve?: Resolve;
 	signal?: AbortSignal;
 	updates?: UpdateOptions<T, I, Resolve>;
-	onLateResults?: (evt: LateResultsEvent) => void | Promise<void>;
+	outOfOrder?:
+		| {
+				mode?: "drop";
+				handle?: (
+					evt: LateResultsEvent<"drop">,
+					helpers: LateResultsHelpers<"drop">,
+				) => void | Promise<void>;
+		  }
+		| {
+				mode: "queue";
+				handle?: (
+					evt: LateResultsEvent<"queue">,
+					helpers: LateResultsHelpers<"queue">,
+				) => void | Promise<void>;
+		  };
 	/**
 	 * Controls iterator liveness after batches are consumed.
 	 * - 'onEmpty' (default): close when no more results
@@ -2649,15 +2683,40 @@ export class DocumentIndex<
 		if (remoteOptions !== originalRemote) {
 			options = Object.assign({}, options, { remote: remoteOptions });
 		}
+		const outOfOrderMode: OutOfOrderMode = options?.outOfOrder?.mode ?? "drop";
 
 		let resolve = options?.resolve !== false;
-		if (
-			!(queryRequestCoerced instanceof types.IterationRequest) &&
+		const wantsReplication =
 			options?.remote &&
 			typeof options.remote !== "boolean" &&
-			(options.remote.replicate || pushUpdates) &&
+			options.remote.replicate;
+
+		if (
+			!(queryRequestCoerced instanceof types.IterationRequest) &&
+			pushUpdates
+		) {
+			// Push streaming only works on IterationRequest; reject legacy compat and upgrade other callers.
+			if (this.compatibility !== undefined) {
+				throw new Error(
+					"updates.push requires IterationRequest support; not available when compatibility is set",
+				);
+			}
+			queryRequestCoerced = new types.IterationRequest({
+				query: queryRequestCoerced.query,
+				sort: queryRequestCoerced.sort,
+				fetch: queryRequestCoerced.fetch,
+				resolve,
+				pushUpdates,
+				mergeUpdates: mergePolicy?.merge ? true : undefined,
+			});
+			resolve =
+				(queryRequestCoerced as types.IterationRequest).resolve !== false;
+		} else if (
+			!(queryRequestCoerced instanceof types.IterationRequest) &&
+			wantsReplication &&
 			options?.resolve !== false
 		) {
+			// Legacy requests can't carry replicate=true; swap to indexed search so replication intent is preserved.
 			if (
 				(queryRequest instanceof types.SearchRequestIndexed === false &&
 					this.compatibility == null) ||
@@ -3496,9 +3555,107 @@ export class DocumentIndex<
 		let fetchedFirstForRemote: Set<string> | undefined = undefined;
 
 		let updateDeferred: ReturnType<typeof pDefer> | undefined;
-		const onLateResults =
-			typeof options?.onLateResults === "function"
-				? options.onLateResults
+		const onLateResultsQueue =
+			options?.outOfOrder?.mode === "queue" &&
+			typeof options?.outOfOrder?.handle === "function"
+				? (options.outOfOrder.handle as (
+						evt: LateResultsEvent<"queue">,
+						helpers: LateResultsHelpers<"queue">,
+					) => void | Promise<void>)
+				: undefined;
+		const onLateResultsDrop =
+			options?.outOfOrder?.mode === "queue"
+				? undefined
+				: typeof options?.outOfOrder?.handle === "function"
+					? (options.outOfOrder.handle as (
+							evt: LateResultsEvent<"drop">,
+							helpers: LateResultsHelpers<"drop">,
+						) => void | Promise<void>)
+					: undefined;
+		const normalizeLateItems = (
+			items?:
+				| {
+						indexed: WithContext<I>;
+						context: types.Context;
+						from: PublicSignKey;
+						value?: types.ResultTypeFromRequest<R, T, I> | I;
+				  }[]
+				| undefined,
+		): LateResultsItem[] | undefined => {
+			if (!items) return undefined;
+			return items.map((item) => {
+				const ctx = item.context || item.indexed.__context;
+				let value = (item.value ?? item.indexed) as any;
+				if (value && ctx && value.__context == null) {
+					value.__context = ctx;
+				}
+				if (value && item.indexed && value.__indexed == null) {
+					value.__indexed = item.indexed;
+				}
+				return {
+					indexed: item.indexed,
+					context: ctx,
+					from: item.from,
+					value,
+				};
+			});
+		};
+
+		const notifyLateResults =
+			onLateResultsQueue || onLateResultsDrop
+				? (
+						amount: number,
+						peer?: PublicSignKey,
+						items?: {
+							indexed: WithContext<I>;
+							context: types.Context;
+							from: PublicSignKey;
+							value?: types.ResultTypeFromRequest<R, T, I> | I;
+						}[],
+					) => {
+						if (amount <= 0) {
+							return;
+						}
+						unsetDone();
+						const payload = items ? normalizeLateItems(items) : undefined;
+						if (outOfOrderMode === "queue" && onLateResultsQueue) {
+							const normalized =
+								payload ??
+								([] as LateResultsEvent<"queue", LateResultsItem>["items"]);
+							const collector = () =>
+								Promise.resolve(
+									normalized as LateResultsEvent<
+										"queue",
+										LateResultsItem
+									>["items"],
+								);
+							onLateResultsQueue(
+								{
+									amount,
+									peer,
+									items: normalized as LateResultsEvent<
+										"queue",
+										LateResultsItem
+									>["items"],
+								},
+								{ collect: collector },
+							);
+							return;
+						}
+						if (onLateResultsDrop) {
+							const collector = () =>
+								Promise.resolve(
+									payload as LateResultsEvent<"drop", LateResultsItem>["items"],
+								);
+							onLateResultsDrop(
+								{
+									amount,
+									peer,
+								},
+								{ collect: collector },
+							);
+						}
+					}
 				: undefined;
 		const runNotify = (reason: UpdateReason) => {
 			if (!updateCallbacks?.notify) {
@@ -3594,6 +3751,18 @@ export class DocumentIndex<
 					return;
 				}
 
+				const collectLateItems =
+					outOfOrderMode !== "drop" && !!notifyLateResults;
+				const lateResults = collectLateItems
+					? ([] as {
+							indexed: WithContext<I>;
+							context: types.Context;
+							from: PublicSignKey;
+							value?: types.ResultTypeFromRequest<R, T, I> | I;
+						}[])
+					: undefined;
+				let lateCount = 0;
+
 				for (const result of results.results) {
 					const indexKey = indexerTypes.toId(
 						this.indexByResolver(result.value),
@@ -3612,18 +3781,28 @@ export class DocumentIndex<
 							indexedPlaceholders?.delete(indexKey);
 							continue;
 						}
-						if (visited.has(indexKey)) {
-							continue;
-						}
-						visited.add(indexKey);
 						const indexed = await this.resolveIndexed<R>(
 							result,
 							results.results as types.ResultTypeFromRequest<R, T, I>[],
 						);
-						if (isLateResult(indexed)) {
-							onLateResults?.({ amount: 1, peer: from });
+						const late = isLateResult(indexed);
+						if (late) {
+							lateCount++;
+							lateResults?.push({
+								indexed,
+								context: result.context,
+								from: from!,
+								value: result.value as types.ResultTypeFromRequest<R, T, I>,
+							});
+							if (outOfOrderMode === "drop") {
+								visited.add(indexKey);
+								continue; // don't buffer late push results
+							}
+						}
+						if (visited.has(indexKey)) {
 							continue;
 						}
+						visited.add(indexKey);
 						buffer.push({
 							value: result.value as types.ResultTypeFromRequest<R, T, I>,
 							context: result.context,
@@ -3631,18 +3810,28 @@ export class DocumentIndex<
 							indexed,
 						});
 					} else {
-						if (visited.has(indexKey) && !indexedPlaceholders?.has(indexKey)) {
-							continue;
-						}
-						visited.add(indexKey);
 						const indexed = coerceWithContext(
 							result.indexed || result.value,
 							result.context,
 						);
-						if (isLateResult(indexed)) {
-							onLateResults?.({ amount: 1, peer: from });
+						const late = isLateResult(indexed);
+						if (late) {
+							lateCount++;
+							lateResults?.push({
+								indexed,
+								context: result.context,
+								from: from!,
+								value: result.value,
+							});
+							if (outOfOrderMode === "drop") {
+								visited.add(indexKey);
+								continue; // don't buffer late push results
+							}
+						}
+						if (visited.has(indexKey) && !indexedPlaceholders?.has(indexKey)) {
 							continue;
 						}
+						visited.add(indexKey);
 						const placeholder = {
 							value: result.value,
 							context: result.context,
@@ -3654,6 +3843,14 @@ export class DocumentIndex<
 					}
 				}
 
+				if (lateCount > 0) {
+					notifyLateResults?.(
+						lateCount,
+						from,
+						collectLateItems ? lateResults : undefined,
+					);
+				}
+
 				peerBufferMap.set(peerHash, {
 					buffer,
 					// Prefetched batches should not claim remote pending counts;
@@ -3859,10 +4056,6 @@ export class DocumentIndex<
 									continue;
 								}
 							}
-							if (isLateResult(indexedCandidate)) {
-								onLateResults?.({ amount: 1 });
-								continue;
-							}
 							const id = indexerTypes.toId(
 								this.indexByResolver(indexedCandidate),
 							).primitive;
@@ -3878,6 +4071,20 @@ export class DocumentIndex<
 								continue;
 							}
 							if (visited.has(id)) continue; // already presented
+							const wasLate = isLateResult(indexedCandidate);
+							if (wasLate) {
+								notifyLateResults?.(1, this.node.identity.publicKey, [
+									{
+										indexed: indexedCandidate,
+										context: added.__context,
+										from: this.node.identity.publicKey,
+										value: resolve ? (added as any) : indexedCandidate,
+									},
+								]);
+								if (outOfOrderMode === "drop") {
+									continue;
+								}
+							}
 							visited.add(id);
 							const valueForBuffer = resolve
 								? (added as any)
@@ -3954,7 +4161,7 @@ export class DocumentIndex<
 							fetchedFirstForRemote,
 						});
 						await fetchPromise;
-						if (onLateResults) {
+						if (onLateResultsQueue || onLateResultsDrop) {
 							const pending = peerBufferMap.get(hash)?.buffer;
 							if (pending && pending.length > 0) {
 								if (lastValueInOrder) {
@@ -3971,10 +4178,21 @@ export class DocumentIndex<
 										(x) => x === lastValueInOrder,
 									);
 									if (lateResults > 0) {
-										onLateResults({ amount: lateResults });
+										const lateItems =
+											outOfOrderMode === "queue"
+												? results.slice(
+														0,
+														Math.min(lateResults, results.length),
+													)
+												: undefined;
+										notifyLateResults?.(lateResults, pk, lateItems);
 									}
 								} else {
-									onLateResults({ amount: pending.length });
+									notifyLateResults?.(
+										pending.length,
+										pk,
+										outOfOrderMode === "queue" ? pending : undefined,
+									);
 								}
 							}
 						}
@@ -4032,11 +4250,11 @@ export class DocumentIndex<
 					warn("Failed to refresh iterator pending state", error);
 				}
 
-				let pendingCount = 0;
+				let total = 0;
 				for (const buffer of peerBufferMap.values()) {
-					pendingCount += buffer.kept + buffer.buffer.length;
+					total += buffer.kept + buffer.buffer.length;
 				}
-				return pendingCount;
+				return total;
 			},
 			all: async () => {
 				drain = true;