@peerbit/document 12.1.2 → 12.2.0-3333888

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/document",
-  "version": "12.1.2",
+  "version": "12.2.0-3333888",
   "description": "Document store implementation",
   "type": "module",
   "sideEffects": false,
@@ -57,32 +57,32 @@
     "@libp2p/interface": "^3.1.0",
     "@libp2p/tcp": "^11.0.2",
     "@multiformats/multiaddr": "^13.0.1",
+    "@peerbit/cache": "2.2.0-3333888",
+    "@peerbit/crypto": "2.4.1-3333888",
+    "@peerbit/logger": "2.0.0-3333888",
+    "@peerbit/log": "5.0.6-3333888",
+    "@peerbit/pubsub": "4.1.3-3333888",
+    "@peerbit/program": "5.6.0-3333888",
+    "@peerbit/rpc": "5.4.15-3333888",
+    "@peerbit/shared-log": "12.2.0-3333888",
+    "@peerbit/indexer-interface": "2.1.1-3333888",
+    "@peerbit/indexer-simple": "1.2.2-3333888",
+    "@peerbit/indexer-sqlite3": "2.1.0-3333888",
+    "@peerbit/document-interface": "3.2.0-3333888",
+    "@peerbit/indexer-cache": "0.2.2-3333888",
+    "@peerbit/stream-interface": "5.3.1-3333888",
     "p-defer": "^4.0.0",
-    "uint8arrays": "^5.1.0",
-    "@peerbit/cache": "2.2.0",
-    "@peerbit/crypto": "2.4.1",
-    "@peerbit/logger": "2.0.0",
-    "@peerbit/log": "5.0.5",
-    "@peerbit/pubsub": "4.1.3",
-    "@peerbit/program": "5.5.2",
-    "@peerbit/rpc": "5.4.14",
-    "@peerbit/shared-log": "12.1.3",
-    "@peerbit/indexer-interface": "2.1.1",
-    "@peerbit/indexer-simple": "1.2.2",
-    "@peerbit/indexer-sqlite3": "2.0.2",
-    "@peerbit/indexer-cache": "0.2.2",
-    "@peerbit/stream-interface": "5.3.1",
-    "@peerbit/document-interface": "3.1.14"
+    "uint8arrays": "^5.1.0"
   },
   "devDependencies": {
+    "@peerbit/test-utils": "2.3.15-3333888",
+    "@peerbit/time": "2.3.0-3333888",
+    "peerbit": "4.4.15-3333888",
     "@types/pidusage": "^2.0.5",
     "pidusage": "^4.0.1",
     "uuid": "^10.0.0",
     "@libp2p/websockets": "^10.1.0",
-    "@chainsafe/libp2p-noise": "^17.0.0",
-    "@peerbit/test-utils": "2.3.14",
-    "peerbit": "4.4.14",
-    "@peerbit/time": "2.3.0"
+    "@chainsafe/libp2p-noise": "^17.0.0"
   },
   "scripts": {
     "clean": "aegir clean",

package/src/index.ts

@@ -9,6 +9,7 @@ export type {
 	WithIndexedContext,
 	WithIndexed,
 	OpenOptions,
+	GetOptions,
 	QueryOptions,
 	RemoteQueryOptions,
 	ResultsIterator,
@@ -40,3 +41,10 @@ export {
 	createDocumentDomainFromProperty,
 } from "./domain.js";
 export * from "./events.js";
+export type {
+	DocumentsLike,
+	DocumentsLikeCountOptions,
+	DocumentsLikeIndex,
+	DocumentsLikeQuery,
+	DocumentsLikeWaitForOptions,
+} from "./like.js";

package/src/like.ts

@@ -0,0 +1,132 @@
+import type {
+	Context,
+	IterationRequest,
+	ResultIndexedValue,
+	ResultValue,
+	Results,
+	SearchRequest,
+	SearchRequestIndexed,
+} from "@peerbit/document-interface";
+import type * as indexerTypes from "@peerbit/indexer-interface";
+import type { Program } from "@peerbit/program";
+import type { SharedLogLike } from "@peerbit/shared-log";
+import type { PeerRefs } from "@peerbit/stream-interface";
+import type {
+	GetOptions,
+	QueryOptions,
+	ReachScope,
+	ResultsIterator,
+	SearchOptions,
+	ValueTypeFromRequest,
+	WithContext,
+	WithIndexedContext,
+} from "./search.js";
+import type { CountEstimate } from "./program.js";
+
+export type DocumentsLikeQuery =
+	| SearchRequest
+	| SearchRequestIndexed
+	| IterationRequest
+	| {
+			query?: indexerTypes.QueryLike | indexerTypes.Query[];
+			sort?: indexerTypes.SortLike | indexerTypes.Sort | indexerTypes.Sort[];
+	  };
+
+export type DocumentsLikeWaitForOptions = {
+	seek?: "any" | "present";
+	signal?: AbortSignal;
+	timeout?: number;
+};
+
+export type DocumentsLikeExactCountOptions = {
+	query?: indexerTypes.Query | indexerTypes.QueryLike;
+	approximate?: false | undefined;
+};
+
+export type DocumentsLikeApproximateCountOptions = {
+	query?: indexerTypes.Query | indexerTypes.QueryLike;
+	approximate: true | { scope?: ReachScope };
+};
+
+export type DocumentsLikeCountOptions =
+	| DocumentsLikeExactCountOptions
+	| DocumentsLikeApproximateCountOptions;
+
+export type DocumentsLikeIndex<T, I, D = any> = {
+	get: <Resolve extends boolean | undefined = true>(
+		id: indexerTypes.Ideable | indexerTypes.IdKey,
+		options?: GetOptions<T, I, D, Resolve>,
+	) => Promise<ValueTypeFromRequest<Resolve, T, I> | undefined>;
+	getDetailed: <Resolve extends boolean | undefined = true>(
+		id: indexerTypes.IdKey | indexerTypes.IdPrimitive,
+		options?: QueryOptions<T, I, D, Resolve>,
+	) => Promise<
+		| Results<
+				Resolve extends false
+					? ResultIndexedValue<WithContext<I>>
+					: ResultValue<WithIndexedContext<T, I>>
+		  >[]
+		| undefined
+	>;
+	resolveId: (value: any) => indexerTypes.IdKey;
+	iterate: <Resolve extends boolean | undefined = true>(
+		query?: DocumentsLikeQuery,
+		options?: QueryOptions<T, I, D, Resolve>,
+	) => ResultsIterator<ValueTypeFromRequest<Resolve, T, I>>;
+	search: <Resolve extends boolean | undefined = true>(
+		query: DocumentsLikeQuery,
+		options?: SearchOptions<T, I, D, Resolve>,
+	) => Promise<ValueTypeFromRequest<Resolve, T, I>[]>;
+	getSize: () => Promise<number> | number;
+	waitFor: (
+		peers: PeerRefs,
+		options?: DocumentsLikeWaitForOptions,
+	) => Promise<string[]>;
+	wrappedIndexedType?: new (value: I, context: Context) => WithContext<I>;
+	index?: {
+		count?: (options?: indexerTypes.CountOptions) => Promise<number> | number;
+		getSize?: () => Promise<number> | number;
+		get?: (
+			id: indexerTypes.IdKey,
+			options?: { shape: indexerTypes.Shape },
+		) =>
+			| Promise<indexerTypes.IndexedResult<WithContext<I>> | undefined>
+			| indexerTypes.IndexedResult<WithContext<I>>
+			| undefined;
+		iterate?: (
+			request?: indexerTypes.IterateOptions,
+			options?: { shape?: indexerTypes.Shape; reference?: boolean },
+		) => indexerTypes.IndexIterator<
+			WithContext<I>,
+			indexerTypes.Shape | undefined
+		>;
+		put?: (value: WithContext<I>) => Promise<void> | void;
+	};
+};
+
+export type DocumentsLike<T, I, D = any> = {
+	closed?: boolean;
+	events: EventTarget;
+	changes: EventTarget;
+	index: DocumentsLikeIndex<T, I, D>;
+	log: SharedLogLike<any>;
+	put(doc: T, options?: unknown): Promise<unknown>;
+	get: (
+		id: indexerTypes.Ideable | indexerTypes.IdKey,
+		options?: Omit<GetOptions<T, I, D, true | undefined>, "resolve">,
+	) => Promise<T | undefined>;
+	del(
+		id: indexerTypes.Ideable | indexerTypes.IdKey,
+		options?: unknown,
+	): Promise<unknown>;
+	count: {
+		(options?: DocumentsLikeExactCountOptions): Promise<number>;
+		(options: DocumentsLikeApproximateCountOptions): Promise<CountEstimate>;
+	};
+	waitFor: (
+		peers: PeerRefs,
+		options?: DocumentsLikeWaitForOptions,
+	) => Promise<string[]>;
+	recover: () => Promise<void>;
+	close: (from?: Program<any, any>) => Promise<boolean | void>;
+};

package/src/program.ts

@@ -42,6 +42,7 @@ import {
 	type CanRead,
 	type CanSearch,
 	DocumentIndex,
+	type GetOptions,
 	type PrefetchOptions,
 	type ReachScope,
 	type TransformOptions,
@@ -175,6 +176,10 @@ export class Documents<
 		return this._index;
 	}
 
+	get changes() {
+		return this.events;
+	}
+
 	private async maybeSubprogramOpen(value: T & Program): Promise<T & Program> {
 		if (await this.canOpen!(value)) {
 			return (await this.node.open(value, {
@@ -300,10 +305,18 @@ export class Documents<
 			trim: options?.log?.trim,
 			replicate: options?.replicate,
 			replicas: options?.replicas,
+			respondToIHaveTimeout: options?.respondToIHaveTimeout,
+			sync: options?.sync,
+			syncronizer: options?.syncronizer,
+			timeUntilRoleMaturity: options?.timeUntilRoleMaturity,
+			waitForReplicatorTimeout: options?.waitForReplicatorTimeout,
+			waitForPruneDelay: options?.waitForPruneDelay,
+			distributionDebounceTime: options?.distributionDebounceTime,
 			domain: (options?.domain
 				? (log: any) => options.domain!(this)
 				: undefined) as any, /// TODO types,
 			compatibility: logCompatiblity,
+			eagerBlocks: options?.eagerBlocks,
 			keep: keepFunction,
 		});
 	}
@@ -333,7 +346,7 @@ export class Documents<
 			: history;
 	}
 
-	async canAppend(
+	protected async canAppend(
 		entry: Entry<Operation>,
 		reference?: { document: T; operation: PutOperation },
 	): Promise<boolean> {
@@ -391,7 +404,7 @@ export class Documents<
 		return true;
 	}
 
-	async _canAppend(
+	protected async _canAppend(
 		entry: Entry<Operation>,
 		reference?: { document: T; operation: PutOperation },
 	): Promise<PutOperation | DeleteOperation | false> {
@@ -604,11 +617,22 @@ export class Documents<
 		return appended;
 	}
 
+	public async get(
+		id: indexerTypes.Ideable | indexerTypes.IdKey,
+		options?: Omit<GetOptions<T, I, D, true | undefined>, "resolve">,
+	): Promise<T | undefined> {
+		const resolved = await this.index.get(id, {
+			...(options ?? {}),
+			resolve: true,
+		} as GetOptions<T, I, D, true>);
+		return resolved ? (resolved as T) : undefined;
+	}
+
 	async del(
-		id: indexerTypes.Ideable,
+		id: indexerTypes.Ideable | indexerTypes.IdKey,
 		options?: SharedAppendOptions<Operation>,
 	) {
-		const key = indexerTypes.toId(id);
+		const key = id instanceof indexerTypes.IdKey ? id : indexerTypes.toId(id);
 		const existing = (
 			await this._index.getDetailed(key, {
 				resolve: false,

package/src/search.ts

@@ -2235,7 +2235,7 @@ export class DocumentIndex<
 			// give queries higher priority than other "normal" data activities
 			// without this, we might have a scenario that a peer joina  network with large amount of data to be synced, but can not query anything before that is done
 			// this will lead to bad UX as you usually want to list/expore whats going on before doing any replication work
-			remote.priority = 1;
+			remote.priority = 2;
 		}
 
 		if (!local && !remote) {
@@ -3728,9 +3728,13 @@ export class DocumentIndex<
 		const keepRemoteAlive = keepOpen && remoteOptions !== false;
 
 		if (queryRequestCoerced instanceof types.IterationRequest) {
-			queryRequestCoerced.resolve = resolve;
+			// If replication is requested, prefer fetching indexed results (with `entries`)
+			// even when the caller wants resolved documents. This allows `replicate(...)` to
+			// join using the provided `Entry` objects instead of doing per-head block fetches.
+			const requestResolve = resolve && !replicate;
+			queryRequestCoerced.resolve = requestResolve;
 			queryRequestCoerced.fetch = queryRequestCoerced.fetch ?? 10;
-			const replicateFlag = !resolve && replicate ? true : false;
+			const replicateFlag = !requestResolve && replicate ? true : false;
 			queryRequestCoerced.replicate = replicateFlag;
 			const ttlSource =
 				typeof remoteOptions === "object" &&
@@ -4251,19 +4255,26 @@ export class DocumentIndex<
 			}
 		};
 
-		return {
-			close,
-			next,
-			done: doneFn,
-			pending: async () => {
-				try {
-					await fetchPromise;
-					if (!done && keepRemoteAlive) {
-						await fetchAtLeast(1);
+			return {
+				close,
+				next,
+				done: doneFn,
+				pending: async () => {
+					try {
+						await fetchPromise;
+						// In push-update mode, remotes will stream new results proactively.
+						// After the iterator has been primed (`first === true`), calling
+						// `fetchAtLeast(1)` from `pending()` can double-count by pulling from
+						// the remote iterator while we also have pushed results buffered locally.
+						//
+						// We still need to prime the iterator at least once so `pending()` is meaningful
+						// even before the first `next(...)` call.
+						if (!done && keepRemoteAlive && (!pushUpdates || !first)) {
+							await fetchAtLeast(1);
+						}
+					} catch (error) {
+						warn("Failed to refresh iterator pending state", error);
 					}
-				} catch (error) {
-					warn("Failed to refresh iterator pending state", error);
-				}
 
 				let total = 0;
 				for (const buffer of peerBufferMap.values()) {
@@ -4273,10 +4284,11 @@ export class DocumentIndex<
 			},
 			all: async () => {
 				drain = true;
+				const drainBatchSize = replicate ? 1000 : 100;
 				let result: ValueTypeFromRequest<Resolve, T, I>[] = [];
 				let c = 0;
 				while (doneFn() !== true) {
-					let batch = await next(100);
+					let batch = await next(drainBatchSize);
 					c += batch.length;
 					if (c > WARNING_WHEN_ITERATING_FOR_MORE_THAN) {
 						warn(
@@ -4304,9 +4316,10 @@ export class DocumentIndex<
 			},
 			[Symbol.asyncIterator]: async function* () {
 				drain = true;
+				const drainBatchSize = replicate ? 1000 : 100;
 				let c = 0;
 				while (doneFn() !== true) {
-					const batch = await next(100);
+					const batch = await next(drainBatchSize);
 					c += batch.length;
 					if (c > WARNING_WHEN_ITERATING_FOR_MORE_THAN) {
 						warn(