@peerbit/document 7.0.13 → 7.0.14-ccaf4f4

package/package.json

@@ -1,76 +1,76 @@
 {
-	"name": "@peerbit/document",
-	"version": "7.0.13",
-	"description": "Document store implementation",
-	"type": "module",
-	"sideEffects": false,
-	"types": "./dist/src/index.d.ts",
-	"typesVersions": {
-		"*": {
-			"*": [
-				"*",
-				"dist/*",
-				"dist/src/*",
-				"dist/src/*/index"
-			],
-			"src/*": [
-				"*",
-				"dist/*",
-				"dist/src/*",
-				"dist/src/*/index"
-			]
-		}
-	},
-	"files": [
-		"src",
-		"dist",
-		"!dist/e2e",
-		"!dist/test",
-		"!**/*.tsbuildinfo"
-	],
-	"exports": {
-		".": {
-			"types": "./dist/src/index.d.ts",
-			"import": "./dist/src/index.js"
-		}
-	},
-	"eslintConfig": {
-		"extends": "peerbit",
-		"parserOptions": {
-			"project": true,
-			"sourceType": "module"
-		},
-		"ignorePatterns": [
-			"!.aegir.js",
-			"test/ts-use",
-			"*.d.ts"
-		]
-	},
-	"publishConfig": {
-		"access": "public"
-	},
-	"scripts": {
-		"clean": "aegir clean",
-		"build": "aegir build --no-bundle",
-		"test": "aegir test --target node",
-		"lint": "aegir lint"
-	},
-	"author": "dao.xyz",
-	"license": "MIT",
-	"dependencies": {
-		"@dao-xyz/borsh": "^5.2.3",
-		"@peerbit/program": "5.0.1",
-		"@peerbit/rpc": "5.0.7",
-		"@peerbit/shared-log": "9.0.9",
-		"@peerbit/indexer-interface": "^1.0.2",
-		"@peerbit/indexer-simple": "^1.0.3",
-		"@peerbit/indexer-sqlite3": "^1.0.5",
-		"@peerbit/document-interface": "^1.0.3"
-	},
-	"devDependencies": {
-		"@peerbit/test-utils": "2.0.41",
-		"@peerbit/time": "2.0.7",
-		"@types/pidusage": "^2.0.5",
-		"pidusage": "^3.0.2"
-	}
+  "name": "@peerbit/document",
+  "version": "7.0.14-ccaf4f4",
+  "description": "Document store implementation",
+  "type": "module",
+  "sideEffects": false,
+  "types": "./dist/src/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "*": [
+        "*",
+        "dist/*",
+        "dist/src/*",
+        "dist/src/*/index"
+      ],
+      "src/*": [
+        "*",
+        "dist/*",
+        "dist/src/*",
+        "dist/src/*/index"
+      ]
+    }
+  },
+  "files": [
+    "src",
+    "dist",
+    "!dist/e2e",
+    "!dist/test",
+    "!**/*.tsbuildinfo"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "import": "./dist/src/index.js"
+    }
+  },
+  "eslintConfig": {
+    "extends": "peerbit",
+    "parserOptions": {
+      "project": true,
+      "sourceType": "module"
+    },
+    "ignorePatterns": [
+      "!.aegir.js",
+      "test/ts-use",
+      "*.d.ts"
+    ]
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "clean": "aegir clean",
+    "build": "aegir build --no-bundle",
+    "test": "aegir test --target node",
+    "lint": "aegir lint"
+  },
+  "author": "dao.xyz",
+  "license": "MIT",
+  "dependencies": {
+    "@dao-xyz/borsh": "^5.2.3",
+    "@peerbit/program": "5.0.1-ccaf4f4",
+    "@peerbit/rpc": "5.0.7-ccaf4f4",
+    "@peerbit/shared-log": "9.0.10-ccaf4f4",
+    "@peerbit/indexer-interface": "1.0.2-ccaf4f4",
+    "@peerbit/indexer-simple": "1.0.3-ccaf4f4",
+    "@peerbit/indexer-sqlite3": "1.0.5-ccaf4f4",
+    "@peerbit/document-interface": "1.0.3-ccaf4f4"
+  },
+  "devDependencies": {
+    "@peerbit/test-utils": "2.0.41-ccaf4f4",
+    "@peerbit/time": "2.0.7-ccaf4f4",
+    "@types/pidusage": "^2.0.5",
+    "pidusage": "^3.0.2"
+  }
 }

package/src/index.ts

@@ -2,14 +2,10 @@ export * from "@peerbit/indexer-interface";
 export * from "./program.js";
 export type {
 	CanRead,
-	BORSH_ENCODING_OPERATION,
 	CanSearch,
-	DeleteOperation,
 	DocumentIndex,
 	IDocumentWithContext,
 	OpenOptions,
-	Operation,
-	PutOperation,
 	QueryOptions,
 	RemoteQueryOptions,
 	ResultsIterator,
@@ -18,5 +14,6 @@ export type {
 	TransformerAsConstructor,
 	TransformerAsFunction,
 } from "./search.js";
+export * from "./operation.js";
 export { MAX_BATCH_SIZE as MAX_DOCUMENT_SIZE } from "./constants.js";
 export { ClosedError } from "@peerbit/program";

package/src/operation.ts

@@ -0,0 +1,106 @@
+import { field, variant } from "@dao-xyz/borsh";
+import * as indexerTypes from "@peerbit/indexer-interface";
+import { BORSH_ENCODING } from "@peerbit/log";
+
+@variant(0)
+export class Operation /* <T> */ {}
+
+export const BORSH_ENCODING_OPERATION = BORSH_ENCODING(Operation);
+
+// @deprecated
+@variant(0)
+export class PutWithKeyOperation extends Operation {
+	@field({ type: "string" })
+	key: string;
+
+	@field({ type: Uint8Array })
+	data: Uint8Array;
+
+	constructor(props: { key: string; data: Uint8Array }) {
+		super();
+		this.key = props.key;
+		this.data = props.data;
+	}
+}
+
+// @deprecated
+/* @variant(1)
+export class PutAllOperation<T> extends Operation<T> {
+    @field({ type: vec(PutOperation) })
+    docs: PutOperation<T>[];
+
+    constructor(props?: { docs: PutOperation<T>[] }) {
+        super();
+        if (props) {
+            this.docs = props.docs;
+        }
+    }
+}
+ */
+
+// @deprecated
+@variant(2)
+export class DeleteByStringKeyOperation extends Operation {
+	@field({ type: "string" })
+	key: string;
+
+	constructor(props: { key: string }) {
+		super();
+		this.key = props.key;
+	}
+
+	toDeleteOperation(): DeleteOperation {
+		return new DeleteOperation({ key: indexerTypes.toId(this.key) });
+	}
+}
+
+export const coerceDeleteOperation = (
+	operation: DeleteOperation | DeleteByStringKeyOperation,
+): DeleteOperation => {
+	return operation instanceof DeleteByStringKeyOperation
+		? operation.toDeleteOperation()
+		: operation;
+};
+
+@variant(3)
+export class PutOperation extends Operation {
+	@field({ type: Uint8Array })
+	data: Uint8Array;
+
+	constructor(props: { data: Uint8Array }) {
+		super();
+		this.data = props.data;
+	}
+}
+
+export const isPutOperation = (
+	operation: Operation,
+): operation is PutOperation | PutWithKeyOperation => {
+	return (
+		operation instanceof PutOperation ||
+		operation instanceof PutWithKeyOperation
+	);
+};
+
+/**
+ * Delete a document at a key
+ */
+@variant(4)
+export class DeleteOperation extends Operation {
+	@field({ type: indexerTypes.IdKey })
+	key: indexerTypes.IdKey;
+
+	constructor(props: { key: indexerTypes.IdKey }) {
+		super();
+		this.key = props.key;
+	}
+}
+
+export const isDeleteOperation = (
+	operation: Operation,
+): operation is DeleteOperation | DeleteByStringKeyOperation => {
+	return (
+		operation instanceof DeleteOperation ||
+		operation instanceof DeleteByStringKeyOperation
+	);
+};

package/src/program.ts

@@ -19,6 +19,7 @@ import {
 import { logger as loggerFn } from "@peerbit/logger";
 import { Program, type ProgramEvents } from "@peerbit/program";
 import {
+	type ReplicationDomain,
 	type SharedAppendOptions,
 	SharedLog,
 	type SharedLogOptions,
@@ -26,17 +27,19 @@ import {
 import { MAX_BATCH_SIZE } from "./constants.js";
 import {
 	BORSH_ENCODING_OPERATION,
-	type CanRead,
-	type CanSearch,
 	DeleteOperation,
-	DocumentIndex,
-	Operation,
+	type Operation,
 	PutOperation,
 	PutWithKeyOperation,
-	type TransformOptions,
 	coerceDeleteOperation,
 	isDeleteOperation,
 	isPutOperation,
+} from "./operation.js";
+import {
+	type CanRead,
+	type CanSearch,
+	DocumentIndex,
+	type TransformOptions,
 } from "./search.js";
 
 const logger = loggerFn({ module: "document" });
@@ -74,7 +77,11 @@ export type CanPerform<T> = (
 	properties: CanPerformOperations<T>,
 ) => MaybePromise<boolean>;
 
-export type SetupOptions<T, I = T> = {
+export type SetupOptions<
+	T,
+	I = T,
+	D extends ReplicationDomain<any, Operation> = any,
+> = {
 	type: AbstractType<T>;
 	canOpen?: (program: T) => MaybePromise<boolean>;
 	canPerform?: CanPerform<T>;
@@ -88,26 +95,29 @@ export type SetupOptions<T, I = T> = {
 		trim?: TrimOptions;
 	};
 	compatibility?: 6;
-} & Exclude<SharedLogOptions<Operation>, "compatibility">;
+} & Exclude<SharedLogOptions<Operation, D>, "compatibility">;
+
+export type ExtractArgs<T> =
+	T extends ReplicationDomain<infer Args, any> ? Args : never;
 
 @variant("documents")
 export class Documents<
 	T,
 	I extends Record<string, any> = T extends Record<string, any> ? T : any,
-> extends Program<SetupOptions<T, I>, DocumentEvents<T> & ProgramEvents> {
+	D extends ReplicationDomain<any, Operation> = any,
+> extends Program<SetupOptions<T, I, D>, DocumentEvents<T> & ProgramEvents> {
 	@field({ type: SharedLog })
-	log: SharedLog<Operation>;
+	log: SharedLog<Operation, D>;
 
 	@field({ type: "bool" })
 	immutable: boolean; // "Can I overwrite a document?"
 
 	@field({ type: DocumentIndex })
-	private _index: DocumentIndex<T, I>;
+	private _index: DocumentIndex<T, I, D>;
 
 	private _clazz!: AbstractType<T>;
 
 	private _optionCanPerform?: CanPerform<T>;
-	private _manuallySynced!: Set<string>;
 	private idResolver!: (any: any) => indexerTypes.IdPrimitive;
 
 	canOpen?: (program: T, entry: Entry<Operation>) => Promise<boolean> | boolean;
@@ -117,7 +127,7 @@ export class Documents<
 	constructor(properties?: {
 		id?: Uint8Array;
 		immutable?: boolean;
-		index?: DocumentIndex<T, I>;
+		index?: DocumentIndex<T, I, ExtractArgs<D>>;
 	}) {
 		super();
 
@@ -126,11 +136,11 @@ export class Documents<
 		this._index = properties?.index || new DocumentIndex();
 	}
 
-	get index(): DocumentIndex<T, I> {
+	get index(): DocumentIndex<T, I, D> {
 		return this._index;
 	}
 
-	async open(options: SetupOptions<T, I>) {
+	async open(options: SetupOptions<T, I, D>) {
 		this._clazz = options.type;
 		this.canOpen = options.canOpen;
 
@@ -144,7 +154,6 @@ export class Documents<
 		}
 
 		this._optionCanPerform = options.canPerform;
-		this._manuallySynced = new Set();
 		const idProperty = options.index?.idProperty || "id";
 		const idResolver =
 			options.id ||
@@ -167,12 +176,10 @@ export class Documents<
 				// here we arrive for all the results we want to persist.
 				// we we need to do here is
 				// 1. add the entry to a list of entries that we should persist through prunes
-				let heads: string[] = [];
-				for (const entry of result.results) {
-					this._manuallySynced.add(entry.context.gid);
-					heads.push(entry.context.head);
-				}
-				return this.log.log.join(heads);
+				await this.log.join(
+					result.results.map((x) => x.context.head),
+					{ replicate: true },
+				);
 			},
 			dbType: this.constructor,
 		});
@@ -185,11 +192,7 @@ export class Documents<
 			trim: options?.log?.trim,
 			replicate: options?.replicate,
 			replicas: options?.replicas,
-			sync: (entry: any) => {
-				// here we arrive when ever a insertion/pruning behaviour processes an entry
-				// returning true means that it should persist
-				return this._manuallySynced.has(entry.gid);
-			},
+			domain: options?.domain,
 
 			// document v6 and below need log compatibility of v8 or below
 			compatibility:
@@ -314,25 +317,47 @@ export class Documents<
 
 				const key = indexerTypes.toId(keyValue);
 
-				const existingDocument = (await this.index.getDetailed(key))?.[0]
-					?.results[0];
+				const existingDocument = (
+					await this.index.getDetailed(key, {
+						local: true,
+						remote: this.immutable,
+					})
+				)?.[0]?.results[0];
 				if (existingDocument && existingDocument.context.head !== entry.hash) {
 					//  econd condition can false if we reset the operation log, while not  resetting the index. For example when doing .recover
 					if (this.immutable) {
-						//Key already exist and this instance Documents can note overrite/edit'
-						return false;
-					}
+						// key already exist but pick the oldest entry
+						// this is because we can not overwrite same id if immutable
+						if (
+							existingDocument.context.created <
+							entry.meta.clock.timestamp.wallTime
+						) {
+							return false;
+						}
 
-					if (entry.meta.next.length !== 1) {
-						return false;
-					}
-					let doc = await this.log.log.get(existingDocument.context.head);
-					if (!doc) {
-						logger.error("Failed to find Document from head");
-						return false;
+						if (entry.meta.next.length > 0) {
+							return false; // can not append to immutable document
+						}
+
+						return putOperation;
+					} else {
+						if (entry.meta.next.length !== 1) {
+							return false;
+						}
+
+						const prevEntry = await this.log.log.entryIndex.get(
+							existingDocument.context.head,
+						);
+						if (!prevEntry) {
+							logger.error(
+								"Failed to find previous entry for document edit: " +
+									entry.hash,
+							);
+							return false;
+						}
+						const referenceHistoryCorrectly = await pointsToHistory(prevEntry);
+						return referenceHistoryCorrectly ? putOperation : false;
 					}
-					const referenceHistoryCorrectly = await pointsToHistory(doc);
-					return referenceHistoryCorrectly ? putOperation : false;
 				} else {
 					if (entry.meta.next.length !== 0) {
 						return false;
@@ -343,7 +368,10 @@ export class Documents<
 					return false;
 				}
 				const existingDocument = (
-					await this.index.getDetailed(operation.key)
+					await this.index.getDetailed(operation.key, {
+						local: true,
+						remote: this.immutable,
+					})
 				)?.[0]?.results[0];
 
 				if (!existingDocument) {
@@ -378,7 +406,10 @@ export class Documents<
 
 	public async put(
 		doc: T,
-		options?: SharedAppendOptions<Operation> & { unique?: boolean },
+		options?: SharedAppendOptions<Operation> & {
+			unique?: boolean;
+			replicate?: boolean;
+		},
 	) {
 		const keyValue = this.idResolver(doc);
 
@@ -399,7 +430,7 @@ export class Documents<
 			: (
 					await this._index.getDetailed(keyValue, {
 						local: true,
-						remote: { sync: true }, // only query remote if we know they exist
+						remote: { replicate: options?.replicate }, // only query remote if we know they exist
 					})
 				)?.[0]?.results[0];
 
@@ -433,6 +464,7 @@ export class Documents<
 			onChange: (change) => {
 				return this.handleChanges(change, { document: doc, operation });
 			},
+			replicate: options?.replicate,
 		});
 
 		return appended;
@@ -446,7 +478,7 @@ export class Documents<
 		const existing = (
 			await this._index.getDetailed(key, {
 				local: true,
-				remote: { sync: true },
+				remote: { replicate: options?.replicate },
 			})
 		)?.[0]?.results[0];
 
@@ -454,6 +486,7 @@ export class Documents<
 			throw new Error(`No entry with key '${key.primitive}' in the database`);
 		}
 
+		const entry = await this._resolveEntry(existing.context.head);
 		return this.log.append(
 			new DeleteOperation({
 				key,
@@ -461,7 +494,7 @@ export class Documents<
 			{
 				...options,
 				meta: {
-					next: [await this._resolveEntry(existing.context.head)],
+					next: [entry],
 					type: EntryType.CUT,
 					...options?.meta,
 				},
@@ -482,7 +515,7 @@ export class Documents<
 		}
 
 		const sortedEntries = [
-			...change.added,
+			...change.added.map((x) => x.entry),
 			...((await Promise.all(
 				change.removed.map((x) =>
 					x instanceof Entry ? x : this.log.log.entryIndex.get(x.hash),
@@ -520,7 +553,6 @@ export class Documents<
 
 					// get index key from value
 					const keyObject = this.idResolver(value);
-
 					const key = indexerTypes.toId(keyObject);
 
 					// document is already updated with more recent entry
@@ -549,8 +581,6 @@ export class Documents<
 					isPutOperation(payload) ||
 					removedSet.has(item.hash)
 				) {
-					this._manuallySynced.delete(item.meta.gid);
-
 					let value: T;
 					let key: indexerTypes.IdKey;