@peerbit/document 9.13.7 → 9.13.8-40eedc0

package/src/search.ts

@@ -2048,15 +2048,35 @@ export class DocumentIndex<
 				buffer: BufferedResult<types.ResultTypeFromRequest<R, T, I> | I, I>[];
 			}
 		> = new Map();
-		const visited = new Set<string | number | bigint>();
+		const visited = new Set<indexerTypes.IdPrimitive>();
+		let indexedPlaceholders:
+			| Map<
+					indexerTypes.IdPrimitive,
+					BufferedResult<types.ResultTypeFromRequest<R, T, I> | I, I>
+			  >
+			| undefined;
+		const ensureIndexedPlaceholders = () => {
+			if (!indexedPlaceholders) {
+				indexedPlaceholders = new Map<
+					string | number | bigint,
+					BufferedResult<types.ResultTypeFromRequest<R, T, I> | I, I>
+				>();
+			}
+			return indexedPlaceholders;
+		};
 
 		let done = false;
 		let drain = false; // if true, close on empty once (overrides manual)
 		let first = false;
 
 		// TODO handle join/leave while iterating
-		const controller = new AbortController();
-
+		const controller: AbortController | undefined = undefined;
+		const ensureController = () => {
+			if (!controller) {
+				return new AbortController();
+			}
+			return controller;
+		};
 		let totalFetchedCounter = 0;
 		let lastValueInOrder:
 			| {
@@ -2081,7 +2101,6 @@ export class DocumentIndex<
 			done = true;
 		};
 		let unsetDone = () => {
-			cleanup();
 			done = false;
 		};
 		let cleanup = () => {
@@ -2127,7 +2146,7 @@ export class DocumentIndex<
 
 			if (options.remote.reach?.discover) {
 				warmupPromise = this.waitFor(options.remote.reach.discover, {
-					signal: controller.signal,
+					signal: ensureController().signal,
 					seek: "present",
 					timeout: waitForTime ?? DEFAULT_TIMEOUT,
 				});
@@ -2147,7 +2166,7 @@ export class DocumentIndex<
 					eager: options.remote.reach?.eager,
 					settle: "any",
 					timeout: waitPolicy.timeout ?? DEFAULT_TIMEOUT,
-					signal: controller.signal,
+					signal: ensureController().signal,
 					onTimeout: waitPolicy.onTimeout,
 				});
 				warmupPromise = warmupPromise
@@ -2204,10 +2223,23 @@ export class DocumentIndex<
 							>[] = peerBufferMap.get(from.hashcode())?.buffer || [];
 
 							for (const result of results.results) {
+								const indexKey = indexerTypes.toId(
+									this.indexByResolver(result.value),
+								).primitive;
 								if (result instanceof types.ResultValue) {
-									const indexKey = indexerTypes.toId(
-										this.indexByResolver(result.value),
-									).primitive;
+									const existingIndexed = indexedPlaceholders?.get(indexKey);
+									if (existingIndexed) {
+										existingIndexed.value =
+											result.value as types.ResultTypeFromRequest<R, T, I>;
+										existingIndexed.context = result.context;
+										existingIndexed.from = from!;
+										existingIndexed.indexed = await this.resolveIndexed<R>(
+											result,
+											results.results,
+										);
+										indexedPlaceholders?.delete(indexKey);
+										continue;
+									}
 									if (visited.has(indexKey)) {
 										continue;
 									}
@@ -2223,23 +2255,25 @@ export class DocumentIndex<
 										),
 									});
 								} else {
-									const indexKey = indexerTypes.toId(
-										this.indexByResolver(result.value),
-									).primitive;
-
-									if (visited.has(indexKey)) {
+									if (
+										visited.has(indexKey) &&
+										!indexedPlaceholders?.has(indexKey)
+									) {
 										continue;
 									}
 									visited.add(indexKey);
-									buffer.push({
+									const indexed = coerceWithContext(
+										result.indexed || result.value,
+										result.context,
+									);
+									const placeholder = {
 										value: result.value,
 										context: result.context,
 										from,
-										indexed: coerceWithContext(
-											result.indexed || result.value,
-											result.context,
-										),
-									});
+										indexed,
+									};
+									buffer.push(placeholder);
+									ensureIndexedPlaceholders().set(indexKey, placeholder);
 								}
 							}
 
@@ -2328,21 +2362,38 @@ export class DocumentIndex<
 										peerBuffer.kept = Number(results.kept);
 
 										for (const result of results.results) {
-											if (
-												visited.has(
-													indexerTypes.toId(this.indexByResolver(result.value))
-														.primitive,
-												)
-											) {
-												continue;
-											}
-											visited.add(
-												indexerTypes.toId(this.indexByResolver(result.value))
-													.primitive,
-											);
-											let indexed: WithContext<I>;
+											const keyPrimitive = indexerTypes.toId(
+												this.indexByResolver(result.value),
+											).primitive;
 											if (result instanceof types.ResultValue) {
-												indexed = await this.resolveIndexed<R>(
+												const existingIndexed =
+													indexedPlaceholders?.get(keyPrimitive);
+												if (existingIndexed) {
+													existingIndexed.value =
+														result.value as types.ResultTypeFromRequest<
+															R,
+															T,
+															I
+														>;
+													existingIndexed.context = result.context;
+													existingIndexed.from = this.node.identity.publicKey;
+													existingIndexed.indexed =
+														await this.resolveIndexed<R>(
+															result,
+															results.results as types.ResultTypeFromRequest<
+																R,
+																T,
+																I
+															>[],
+														);
+													indexedPlaceholders?.delete(keyPrimitive);
+													continue;
+												}
+												if (visited.has(keyPrimitive)) {
+													continue;
+												}
+												visited.add(keyPrimitive);
+												const indexed = await this.resolveIndexed<R>(
 													result,
 													results.results as types.ResultTypeFromRequest<
 														R,
@@ -2350,22 +2401,40 @@ export class DocumentIndex<
 														I
 													>[],
 												);
+												peerBuffer.buffer.push({
+													value: result.value as types.ResultTypeFromRequest<
+														R,
+														T,
+														I
+													>,
+													context: result.context,
+													from: this.node.identity.publicKey,
+													indexed,
+												});
 											} else {
-												indexed = coerceWithContext(
+												if (
+													visited.has(keyPrimitive) &&
+													!indexedPlaceholders?.has(keyPrimitive)
+												) {
+													continue;
+												}
+												visited.add(keyPrimitive);
+												const indexed = coerceWithContext(
 													result.indexed || result.value,
 													result.context,
 												);
+												const placeholder = {
+													value: result.value,
+													context: result.context,
+													from: this.node.identity.publicKey,
+													indexed,
+												};
+												peerBuffer.buffer.push(placeholder);
+												ensureIndexedPlaceholders().set(
+													keyPrimitive,
+													placeholder,
+												);
 											}
-											peerBuffer.buffer.push({
-												value: result.value as types.ResultTypeFromRequest<
-													R,
-													T,
-													I
-												>,
-												context: result.context,
-												from: this.node.identity.publicKey,
-												indexed: indexed,
-											});
 										}
 									}
 								})
@@ -2395,8 +2464,11 @@ export class DocumentIndex<
 								.request(remoteCollectRequest, {
 									...options,
 									signal: options?.signal
-										? AbortSignal.any([options.signal, controller.signal])
-										: controller.signal,
+										? AbortSignal.any([
+												options.signal,
+												ensureController().signal,
+											])
+										: ensureController().signal,
 									priority: 1,
 									mode: new SilentDelivery({ to: [peer], redundancy: 1 }),
 								})
@@ -2436,17 +2508,40 @@ export class DocumentIndex<
 																>
 															>
 														).response.results) {
-															const idPrimitive = indexerTypes.toId(
+															const indexKey = indexerTypes.toId(
 																this.indexByResolver(result.value),
 															).primitive;
-															if (visited.has(idPrimitive)) {
-																continue;
-															}
-															visited.add(idPrimitive);
-
-															let indexed: WithContext<I>;
 															if (result instanceof types.ResultValue) {
-																indexed = await this.resolveIndexed(
+																const existingIndexed =
+																	indexedPlaceholders?.get(indexKey);
+																if (existingIndexed) {
+																	existingIndexed.value =
+																		result.value as types.ResultTypeFromRequest<
+																			R,
+																			T,
+																			I
+																		>;
+																	existingIndexed.context = result.context;
+																	existingIndexed.from = from!;
+																	existingIndexed.indexed =
+																		await this.resolveIndexed(
+																			result,
+																			response.response
+																				.results as types.ResultTypeFromRequest<
+																				R,
+																				T,
+																				I
+																			>[],
+																		);
+																	indexedPlaceholders?.delete(indexKey);
+																	continue;
+																}
+																if (visited.has(indexKey)) {
+																	continue;
+																}
+																visited.add(indexKey);
+
+																const indexed = await this.resolveIndexed(
 																	result,
 																	response.response
 																		.results as types.ResultTypeFromRequest<
@@ -2455,23 +2550,41 @@ export class DocumentIndex<
 																		I
 																	>[],
 																);
+																peerBuffer.buffer.push({
+																	value:
+																		result.value as types.ResultTypeFromRequest<
+																			R,
+																			T,
+																			I
+																		>,
+																	context: result.context,
+																	from: from!,
+																	indexed,
+																});
 															} else {
-																indexed = coerceWithContext(
+																if (
+																	visited.has(indexKey) &&
+																	!indexedPlaceholders?.has(indexKey)
+																) {
+																	continue;
+																}
+																visited.add(indexKey);
+																const indexed = coerceWithContext(
 																	result.value,
 																	result.context,
 																);
+																const placeholder = {
+																	value: result.value,
+																	context: result.context,
+																	from: from!,
+																	indexed,
+																};
+																peerBuffer.buffer.push(placeholder);
+																ensureIndexedPlaceholders().set(
+																	indexKey,
+																	placeholder,
+																);
 															}
-															peerBuffer.buffer.push({
-																value:
-																	result.value as types.ResultTypeFromRequest<
-																		R,
-																		T,
-																		I
-																	>,
-																context: result.context,
-																from: from!,
-																indexed,
-															});
 														}
 													}
 												}),
@@ -2534,6 +2647,10 @@ export class DocumentIndex<
 				const idx = arr.buffer.findIndex((x) => x.value === result.value);
 				if (idx >= 0) {
 					arr.buffer.splice(idx, 1);
+					const consumedId = indexerTypes.toId(
+						this.indexByResolver(result.indexed),
+					).primitive;
+					indexedPlaceholders?.delete(consumedId);
 				}
 			}
 
@@ -2580,7 +2697,9 @@ export class DocumentIndex<
 
 		let cleanupAndDone = () => {
 			cleanup();
-			controller.abort(new AbortError("Iterator closed"));
+			(controller as AbortController | undefined)?.abort(
+				new AbortError("Iterator closed"),
+			);
 			this.prefetch?.accumulator.clear(queryRequestCoerced);
 			this.processCloseIteratorRequest(
 				queryRequestCoerced,
@@ -2751,27 +2870,65 @@ export class DocumentIndex<
 					filtered = await mergePolicy.merge?.filter(evt.detail);
 				}
 				if (filtered) {
+					const changeForCallback: DocumentsChange<T, I> = {
+						added: [],
+						removed: [],
+					};
+					let hasRelevantChange = false;
+
 					// Remove entries that were deleted from all pending structures
 					if (filtered.removed?.length) {
-						// Remove from peer buffers
+						const removedIds = new Set<string | number | bigint>();
+						for (const removed of filtered.removed) {
+							const id = indexerTypes.toId(
+								this.indexByResolver(removed.__indexed),
+							).primitive;
+							removedIds.add(id);
+						}
+						const matchedRemovedIds = new Set<string | number | bigint>();
 						for (const [_peer, entry] of peerBufferMap) {
+							if (entry.buffer.length === 0) {
+								continue;
+							}
 							entry.buffer = entry.buffer.filter((x) => {
 								const id = indexerTypes.toId(
 									this.indexByResolver(x.indexed),
 								).primitive;
-								return !filtered!.removed!.some(
-									(r) =>
-										indexerTypes.toId(this.indexByResolver(r.__indexed))
-											.primitive === id,
-								);
+								if (removedIds.has(id)) {
+									matchedRemovedIds.add(id);
+									indexedPlaceholders?.delete(id);
+									return false;
+								}
+								return true;
 							});
 						}
-						// no non-sorted queues in simplified mode
+						if (matchedRemovedIds.size > 0) {
+							hasRelevantChange = true;
+							for (const removed of filtered.removed) {
+								const id = indexerTypes.toId(
+									this.indexByResolver(removed.__indexed),
+								).primitive;
+								if (matchedRemovedIds.has(id)) {
+									changeForCallback.removed.push(removed);
+								}
+							}
+						}
 					}
 
 					// Add new entries per strategy (sorted-only)
 					if (filtered.added?.length) {
-						const buf = peerBufferMap.get(localHash)!;
+						let buf = peerBufferMap.get(localHash);
+						if (!buf) {
+							const created: {
+								kept: number;
+								buffer: BufferedResult<
+									types.ResultTypeFromRequest<R, T, I> | I,
+									I
+								>[];
+							} = { kept: 0, buffer: [] };
+							peerBufferMap.set(localHash, created);
+							buf = created;
+						}
 						const filterIndex = hasQueryFiltersForUpdates
 							? await createUpdateFilterIndex()
 							: undefined;
@@ -2801,31 +2958,49 @@ export class DocumentIndex<
 							const id = indexerTypes.toId(
 								this.indexByResolver(indexedCandidate),
 							).primitive;
+							const existingIndexed = indexedPlaceholders?.get(id);
+							if (existingIndexed) {
+								if (resolve) {
+									existingIndexed.value = added as any;
+									existingIndexed.context = added.__context;
+									existingIndexed.from = this.node.identity.publicKey;
+									existingIndexed.indexed = indexedCandidate;
+									indexedPlaceholders?.delete(id);
+								}
+								continue;
+							}
 							if (visited.has(id)) continue; // already presented
 							visited.add(id);
 							const valueForBuffer = resolve
 								? (added as any)
 								: indexedCandidate;
-							buf.buffer.push({
+							const placeholder = {
 								value: valueForBuffer,
 								context: added.__context,
 								from: this.node.identity.publicKey,
 								indexed: indexedCandidate,
-							});
+							};
+							buf.buffer.push(placeholder);
+							if (!resolve) {
+								ensureIndexedPlaceholders().set(id, placeholder);
+							}
+							hasRelevantChange = true;
+							changeForCallback.added.push(added);
 						}
-						buf.kept = buf.buffer.length;
 					}
 
-					if (
-						(filtered.added?.length ?? 0) > 0 ||
-						(filtered.removed?.length ?? 0) > 0
-					) {
+					if (hasRelevantChange) {
 						if (!pendingResultsReason) {
 							pendingResultsReason = "change";
 						}
+						if (
+							changeForCallback.added.length > 0 ||
+							changeForCallback.removed.length > 0
+						) {
+							updateCallbacks?.onChange?.(changeForCallback);
+						}
 					}
 				}
-				updateCallbacks?.onChange?.(evt.detail); // TODO only emit if changes were relevant?
 				signalUpdate();
 			};
 
@@ -2866,10 +3041,10 @@ export class DocumentIndex<
 				setTimeout(() => {
 					signalUpdate();
 				}, waitForTime);
-			controller.signal.addEventListener("abort", () => signalUpdate());
+			ensureController().signal.addEventListener("abort", () => signalUpdate());
 			fetchedFirstForRemote = new Set<string>();
 			joinListener = this.attachJoinListener({
-				signal: controller.signal,
+				signal: ensureController().signal,
 				eager: options.remote.reach?.eager,
 				onPeer: async (pk) => {
 					if (done) return;
@@ -2952,11 +3127,11 @@ export class DocumentIndex<
 			next,
 			done: doneFn,
 			pending: () => {
-				let kept = 0;
-				for (const [_, buffer] of peerBufferMap) {
-					kept += buffer.kept;
+				let pendingCount = 0;
+				for (const buffer of peerBufferMap.values()) {
+					pendingCount += buffer.kept + buffer.buffer.length;
 				}
-				return kept; // TODO this should be more accurate
+				return pendingCount;
 			},
 			all: async () => {
 				drain = true;