@peerbit/document 13.0.8 → 13.0.9

package/src/program.ts

@@ -43,9 +43,12 @@ import {
 	type CanSearch,
 	DocumentIndex,
 	type GetOptions,
+	type IndexedContextOnly,
+	INDEX_CONTEXT_SHAPE,
 	type PrefetchOptions,
 	type ReachScope,
 	type TransformOptions,
+	type WithContext,
 	type WithIndexedContext,
 	coerceWithContext,
 	coerceWithIndexed,
@@ -176,6 +179,33 @@ export class Documents<
 		return this._index;
 	}
 
+	private getLocalIndexedContext(
+		key: indexerTypes.IdKey,
+	): Promise<indexerTypes.IndexedResult<IndexedContextOnly<I>> | undefined> {
+		return this._index.index.get(key, {
+			shape: INDEX_CONTEXT_SHAPE,
+		}) as Promise<
+			indexerTypes.IndexedResult<IndexedContextOnly<I>> | undefined
+		>;
+	}
+
+	private getExistingContext(
+		existing:
+			| ResultIndexedValue<WithContext<I>>
+			| indexerTypes.IndexedResult<WithContext<I>>
+			| indexerTypes.IndexedResult<IndexedContextOnly<I>>
+			| null
+			| undefined,
+	): indexerTypes.ReturnTypeFromShape<
+		WithContext<I>,
+		typeof INDEX_CONTEXT_SHAPE
+	>["__context"]
+		| undefined {
+		return existing instanceof ResultIndexedValue
+			? existing.context
+			: existing?.value.__context;
+	}
+
 	get changes() {
 		return this.events;
 	}
@@ -453,20 +483,23 @@ export class Documents<
 
 				const key = indexerTypes.toId(keyValue);
 
-				const existingDocument = (
-					await this.index.getDetailed(key, {
-						resolve: false,
-						local: true,
-						remote: this.immutable ? { strategy: "fallback" } : false,
-					})
-				)?.[0]?.results[0];
-				if (existingDocument && existingDocument.context.head !== entry.hash) {
+				const existingDocument = this.immutable
+					? (
+							await this.index.getDetailed(key, {
+								resolve: false,
+								local: true,
+								remote: { strategy: "fallback" },
+							})
+						)?.[0]?.results[0]
+					: await this.getLocalIndexedContext(key);
+				const existingContext = this.getExistingContext(existingDocument);
+				if (existingContext && existingContext.head !== entry.hash) {
 					//  econd condition can false if we reset the operation log, while not  resetting the index. For example when doing .recover
 					if (this.immutable) {
 						// key already exist but pick the oldest entry
 						// this is because we can not overwrite same id if immutable
 						if (
-							existingDocument.context.created <
+							existingContext.created <
 							entry.meta.clock.timestamp.wallTime
 						) {
 							return false;
@@ -485,7 +518,7 @@ export class Documents<
 							}
 
 							const prevEntry = await this.log.log.entryIndex.get(
-								existingDocument.context.head,
+								existingContext.head,
 							);
 							if (!prevEntry) {
 								logger.error(
@@ -510,19 +543,26 @@ export class Documents<
 				if (entry.meta.next.length !== 1) {
 					return false;
 				}
-				const existingDocument = (
-					await this.index.getDetailed(operation.key, {
-						resolve: false,
-						local: true,
-						remote: this.immutable,
-					})
-				)?.[0]?.results[0];
-
-				if (!existingDocument) {
+				const existingDocument = this.immutable
+					? (
+							await this.index.getDetailed(operation.key, {
+								resolve: false,
+								local: true,
+								remote: true,
+							})
+						)?.[0]?.results[0]
+					: await this.getLocalIndexedContext(
+							operation.key instanceof indexerTypes.IdKey
+								? operation.key
+								: indexerTypes.toId(operation.key),
+						);
+				const existingHead = this.getExistingContext(existingDocument)?.head;
+
+				if (!existingHead) {
 					// already deleted
 					return coerceDeleteOperation(operation); // assume ok
 				}
-				let doc = await this.log.log.get(existingDocument.context.head);
+				let doc = await this.log.log.get(existingHead);
 				if (!doc) {
 					logger.error("Failed to find Document from head");
 					return false;
@@ -560,7 +600,6 @@ export class Documents<
 
 		// type check the key
 		indexerTypes.checkId(keyValue);
-
 		const ser = serialize(doc);
 		if (ser.length > MAX_BATCH_SIZE) {
 			throw new Error(
@@ -570,17 +609,18 @@ export class Documents<
 			);
 		}
 
-		const existingDocument = options?.unique
+		const existingHead = options?.unique
 			? undefined
-			: (
-					await this._index.getDetailed(keyValue, {
-						resolve: false,
-						local: true,
-						remote: options?.checkRemote
-							? { replicate: options?.replicate }
-							: false, // only query remote if we know they exist
-					})
-				)?.[0]?.results[0];
+			: options?.checkRemote
+				? (
+						await this._index.getDetailed(keyValue, {
+							resolve: false,
+							local: true,
+							remote: { replicate: options?.replicate },
+						})
+					)?.[0]?.results[0]?.context.head
+				: (await this.getLocalIndexedContext(indexerTypes.toId(keyValue)))
+						?.value.__context.head;
 
 		let operation: PutOperation | PutWithKeyOperation;
 		if (this.compatibility === 6) {
@@ -601,9 +641,7 @@ export class Documents<
 		const appended = await this.log.append(operation, {
 			...options,
 			meta: {
-				next: existingDocument
-					? [await this._resolveEntry(existingDocument.context.head)]
-					: [],
+				next: existingHead ? [await this._resolveEntry(existingHead)] : [],
 				...options?.meta,
 			},
 			canAppend: (entry) => {
@@ -732,7 +770,7 @@ export class Documents<
 					// if no casual ordering is used, use timestamps to order docs
 					let existing = reference?.unique
 						? null
-						: (await this._index.index.get(key)) || null;
+						: (await this.getLocalIndexedContext(key)) || null;
 					if (!this.strictHistory && existing) {
 						// if immutable use oldest, else use newest
 						let shouldIgnoreChange = this.immutable

package/src/search.ts

@@ -197,6 +197,7 @@ export type RemoteQueryOptions<Q, R, D> = RPCRequestAllOptions<Q, R> & {
 	replicate?: boolean;
 	minAge?: number;
 	throwOnMissing?: boolean;
+	retryMissingResponses?: boolean;
 	strategy?: "fallback";
 	domain?:
 		| {
@@ -404,6 +405,7 @@ const introduceEntries = async <
 	RPCResponse<types.Results<types.ResultTypeFromRequest<R, T, I>>>[]
 > => {
 	const results: RPCResponse<types.Results<any>>[] = [];
+	const replicatedHeads = new Set<string>();
 	for (const response of responses) {
 		if (!response.from) {
 			logger.error("Missing from for response");
@@ -416,7 +418,27 @@ const introduceEntries = async <
 					: r.init(indexedType),
 			);
 			if (typeof options?.remote !== "boolean" && options?.remote?.replicate) {
-				await sync(queryRequest, response.response);
+				const uniqueResults = response.response.results.filter((result) => {
+					const head =
+						result instanceof types.ResultIndexedValue &&
+						result.entries.length > 0
+							? result.entries[0]!.hash
+							: result.context.head;
+					if (replicatedHeads.has(head)) {
+						return false;
+					}
+					replicatedHeads.add(head);
+					return true;
+				});
+				if (uniqueResults.length > 0) {
+					await sync(
+						queryRequest,
+						new types.Results({
+							results: uniqueResults,
+							kept: response.response.kept,
+						}),
+					);
+				}
 			}
 			options?.onResponse &&
 				(await options.onResponse(response.response, response.from!)); // TODO fix types
@@ -524,6 +546,19 @@ export type WithContext<I> = {
 	__context: types.Context;
 } & I;
 
+export const INDEX_CONTEXT_SHAPE = {
+	__context: {
+		created: true,
+		modified: true,
+		head: true,
+		gid: true,
+		size: true,
+	},
+} as const;
+
+export type IndexedContextOnly<I extends Record<string, any>> =
+	indexerTypes.ReturnTypeFromShape<WithContext<I>, typeof INDEX_CONTEXT_SHAPE>;
+
 export type WithIndexed<T, I> = {
 	// experimental, used to quickly get the indexed representation
 	__indexed: I;
@@ -1534,10 +1569,18 @@ export class DocumentIndex<
 		value: T,
 		id: indexerTypes.IdKey,
 		entry: Entry<Operation>,
-		existing: indexerTypes.IndexedResult<WithContext<I>> | null | undefined,
+		existing:
+			| indexerTypes.IndexedResult<WithContext<I>>
+			| indexerTypes.IndexedResult<IndexedContextOnly<I>>
+			| null
+			| undefined,
 	): Promise<{ context: types.Context; indexable: I }> {
 		const existingDefined =
-			existing === undefined ? await this.index.get(id) : existing;
+			existing === undefined
+				? await this.index.get(id, {
+						shape: INDEX_CONTEXT_SHAPE,
+					})
+				: existing;
 		const context = new types.Context({
 			created:
 				existingDefined?.value.__context.created ||
@@ -1547,13 +1590,16 @@ export class DocumentIndex<
 			gid: entry.meta.gid,
 			size: entry.payload.byteLength,
 		});
-		return this.putWithContext(value, id, context);
+		return this.putWithContext(value, id, context, {
+			replace: existingDefined != null,
+		});
 	}
 
 	public async putWithContext(
 		value: T,
 		id: indexerTypes.IdKey,
 		context: types.Context,
+		options?: { replace?: boolean },
 	): Promise<{ context: types.Context; indexable: I }> {
 		const idString = id.primitive;
 		if (
@@ -1582,7 +1628,7 @@ export class DocumentIndex<
 
 		coerceWithContext(value, context);
 
-		await this.index.put(wrappedValueToIndex);
+		await this.index.put(wrappedValueToIndex, undefined, options);
 		return { context, indexable: valueToIndex };
 	}
 
@@ -2580,17 +2626,29 @@ export class DocumentIndex<
 		options?: O,
 	): Promise<ValueTypeFromRequest<Resolve, T, I>[]> {
 		// Set fetch to search size, or max value (default to max u32 (4294967295))
-			const coercedRequest = coerceQuery(
-				queryRequest,
-				options,
-				this.compatibility,
-			);
-			coercedRequest.fetch = coercedRequest.fetch ?? 0xffffffff;
+		const coercedRequest = coerceQuery(
+			queryRequest,
+			options,
+			this.compatibility,
+		);
+		coercedRequest.fetch = coercedRequest.fetch ?? 0xffffffff;
+
+		const searchOptions =
+			typeof options?.remote === "object" &&
+			options.remote.retryMissingResponses === undefined
+				? ({
+						...options,
+						remote: {
+							...options.remote,
+							retryMissingResponses: false,
+						},
+					} as O)
+				: options;
 
-			// Use an iterator so large results respect message size limits.
-			const iterator = this.iterate<Resolve>(coercedRequest, options);
+		// Use an iterator so large results respect message size limits.
+		const iterator = this.iterate<Resolve>(coercedRequest, searchOptions);
 
-			const allResults: ValueTypeFromRequest<Resolve, T, I>[] = [];
+		const allResults: ValueTypeFromRequest<Resolve, T, I>[] = [];
 
 		while (
 			iterator.done() !== true &&
@@ -2870,6 +2928,10 @@ export class DocumentIndex<
 		) {
 			queryRequestCoerced.replicate = true;
 		}
+		const retryMissingResponseGroups =
+			typeof options?.remote === "object"
+				? options.remote.retryMissingResponses ?? true
+				: true;
 
 		indexIteratorLogger.trace("Iterate with options", {
 			query: queryRequestCoerced,
@@ -3216,6 +3278,9 @@ export class DocumentIndex<
 					},
 					onMissingResponses: (error) => {
 						missingResponses = true;
+						if (!retryMissingResponseGroups) {
+							return;
+						}
 						const missingGroups = (error as MissingResponsesError & {
 							missingGroups?: string[][];
 						}).missingGroups;
@@ -3242,7 +3307,7 @@ export class DocumentIndex<
 				fetchOptions?.fetchedFirstForRemote,
 			);
 
-			if (missingResponses) {
+			if (missingResponses && retryMissingResponseGroups) {
 				hasMore = true;
 				unsetDone();
 			}