@peerbit/document 6.0.7 → 7.0.0-3a75d6e

package/dist/src/borsh.js

@@ -0,0 +1,16 @@
+import { getSchema } from "@dao-xyz/borsh";
+export const copySerialization = (sourceClazz, targetClazz) => {
+    const copiedFromAlready = targetClazz["__copiedFrom"] || [];
+    if (copiedFromAlready?.includes(sourceClazz)) {
+        return;
+    }
+    copiedFromAlready.push(sourceClazz);
+    targetClazz["__copiedFrom"] = copiedFromAlready;
+    const targetSchema = getSchema(targetClazz);
+    const sourceSchema = getSchema(sourceClazz);
+    targetSchema.fields = [...sourceSchema.fields, ...targetSchema.fields];
+    targetSchema.variant = sourceSchema.variant;
+    targetSchema.getDependencies =
+        sourceSchema.getDependencies.bind(sourceSchema);
+};
+//# sourceMappingURL=borsh.js.map

package/dist/src/borsh.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"borsh.js","sourceRoot":"","sources":["../../src/borsh.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,WAAgB,EAAE,WAAgB,EAAE,EAAE;IACvE,MAAM,iBAAiB,GAAU,WAAW,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IACnE,IAAI,iBAAiB,EAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9C,OAAO;IACR,CAAC;IAED,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpC,WAAW,CAAC,cAAc,CAAC,GAAG,iBAAiB,CAAC;IAEhD,MAAM,YAAY,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;IAC5C,MAAM,YAAY,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;IAE5C,YAAY,CAAC,MAAM,GAAG,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACvE,YAAY,CAAC,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC;IAC5C,YAAY,CAAC,eAAe;QAC3B,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AAClD,CAAC,CAAC"}

package/dist/src/constants.d.ts

@@ -0,0 +1,2 @@
+export declare const MAX_BATCH_SIZE = 5000000;
+//# sourceMappingURL=constants.d.ts.map

package/dist/src/constants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,cAAc,UAAM,CAAC"}

package/dist/src/constants.js

@@ -0,0 +1,2 @@
+export const MAX_BATCH_SIZE = 5e6;
+//# sourceMappingURL=constants.js.map

package/dist/src/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,CAAC"}

package/dist/src/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./program.js";
+export * from "./search.js";
+export { MAX_BATCH_SIZE as MAX_DOCUMENT_SIZE } from "./constants.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,OAAO,EAAE,cAAc,IAAI,iBAAiB,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/src/index.js

@@ -0,0 +1,4 @@
+export * from "./program.js";
+export * from "./search.js";
+export { MAX_BATCH_SIZE as MAX_DOCUMENT_SIZE } from "./constants.js";
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,OAAO,EAAE,cAAc,IAAI,iBAAiB,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/src/program.d.ts

@@ -0,0 +1,87 @@
+import { type AbstractType } from "@dao-xyz/borsh";
+import * as indexerTypes from "@peerbit/indexer-interface";
+import { type Change, Entry, type ShallowOrFullEntry, type TrimOptions } from "@peerbit/log";
+import { Program, type ProgramEvents } from "@peerbit/program";
+import { type SharedAppendOptions, SharedLog, type SharedLogOptions } from "@peerbit/shared-log";
+import { type CanRead, type CanSearch, DeleteOperation, DocumentIndex, Operation, PutOperation, type TransformOptions } from "./search.js";
+export declare class OperationError extends Error {
+    constructor(message?: string);
+}
+export interface DocumentsChange<T> {
+    added: T[];
+    removed: T[];
+}
+export interface DocumentEvents<T> {
+    change: CustomEvent<DocumentsChange<T>>;
+}
+type MaybePromise<T> = Promise<T> | T;
+type CanPerformPut<T> = {
+    type: "put";
+    value: T;
+    operation: PutOperation;
+    entry: Entry<PutOperation>;
+};
+type CanPerformDelete<T> = {
+    type: "delete";
+    operation: DeleteOperation;
+    entry: Entry<DeleteOperation>;
+};
+export type CanPerformOperations<T> = CanPerformPut<T> | CanPerformDelete<T>;
+export type CanPerform<T> = (properties: CanPerformOperations<T>) => MaybePromise<boolean>;
+export type SetupOptions<T, I = T> = {
+    type: AbstractType<T>;
+    canOpen?: (program: T) => MaybePromise<boolean>;
+    canPerform?: CanPerform<T>;
+    id?: (obj: any) => indexerTypes.IdPrimitive;
+    index?: {
+        canSearch?: CanSearch;
+        canRead?: CanRead<T>;
+        idProperty?: string | string[];
+    } & TransformOptions<T, I>;
+    log?: {
+        trim?: TrimOptions;
+    };
+} & SharedLogOptions<Operation>;
+export declare class Documents<T, I extends Record<string, any> = T extends Record<string, any> ? T : any> extends Program<SetupOptions<T, I>, DocumentEvents<T> & ProgramEvents> {
+    log: SharedLog<Operation>;
+    immutable: boolean;
+    private _index;
+    private _clazz;
+    private _optionCanPerform?;
+    private _manuallySynced;
+    private idResolver;
+    canOpen?: (program: T, entry: Entry<Operation>) => Promise<boolean> | boolean;
+    constructor(properties?: {
+        id?: Uint8Array;
+        immutable?: boolean;
+        index?: DocumentIndex<T, I>;
+    });
+    get index(): DocumentIndex<T, I>;
+    open(options: SetupOptions<T, I>): Promise<void>;
+    recover(): Promise<void>;
+    private _resolveEntry;
+    canAppend(entry: Entry<Operation>, reference?: {
+        document: T;
+        operation: PutOperation;
+    }): Promise<boolean>;
+    _canAppend(entry: Entry<Operation>, reference?: {
+        document: T;
+        operation: PutOperation;
+    }): Promise<PutOperation | DeleteOperation | false>;
+    put(doc: T, options?: SharedAppendOptions<Operation> & {
+        unique?: boolean;
+    }): Promise<{
+        entry: Entry<Operation>;
+        removed: ShallowOrFullEntry<Operation>[];
+    }>;
+    del(id: indexerTypes.Ideable, options?: SharedAppendOptions<Operation>): Promise<{
+        entry: Entry<Operation>;
+        removed: ShallowOrFullEntry<Operation>[];
+    }>;
+    handleChanges(change: Change<Operation>, reference?: {
+        document: T;
+        operation: PutOperation;
+    }): Promise<void>;
+}
+export {};
+//# sourceMappingURL=program.d.ts.map

package/dist/src/program.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"program.d.ts","sourceRoot":"","sources":["../../src/program.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,KAAK,YAAY,EAKjB,MAAM,gBAAgB,CAAC;AAIxB,OAAO,KAAK,YAAY,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EACN,KAAK,MAAM,EACX,KAAK,EAEL,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EACN,KAAK,mBAAmB,EACxB,SAAS,EACT,KAAK,gBAAgB,EACrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAEN,KAAK,OAAO,EACZ,KAAK,SAAS,EACd,eAAe,EACf,aAAa,EACb,SAAS,EACT,YAAY,EACZ,KAAK,gBAAgB,EACrB,MAAM,aAAa,CAAC;AAIrB,qBAAa,cAAe,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG5B;AACD,MAAM,WAAW,eAAe,CAAC,CAAC;IACjC,KAAK,EAAE,CAAC,EAAE,CAAC;IACX,OAAO,EAAE,CAAC,EAAE,CAAC;CACb;AACD,MAAM,WAAW,cAAc,CAAC,CAAC;IAChC,MAAM,EAAE,WAAW,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;CACxC;AAED,KAAK,YAAY,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAEtC,KAAK,aAAa,CAAC,CAAC,IAAI;IACvB,IAAI,EAAE,KAAK,CAAC;IACZ,KAAK,EAAE,CAAC,CAAC;IACT,SAAS,EAAE,YAAY,CAAC;IACxB,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;CAC3B,CAAC;AAEF,KAAK,gBAAgB,CAAC,CAAC,IAAI;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,eAAe,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,oBAAoB,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;AAC7E,MAAM,MAAM,UAAU,CAAC,CAAC,IAAI,CAC3B,UAAU,EAAE,oBAAoB,CAAC,CAAC,CAAC,KAC/B,YAAY,CAAC,OAAO,CAAC,CAAC;AAE3B,MAAM,MAAM,YAAY,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI;IACpC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACtB,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,YAAY,CAAC,OAAO,CAAC,CAAC;IAChD,UAAU,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IAC3B,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,YAAY,CAAC,WAAW,CAAC;IAC5C,KAAK,CAAC,EAAE;QACP,SAAS,CAAC,EAAE,SAAS,CAAC;QACtB,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KAC/B,GAAG,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3B,GAAG,CAAC,EAAE;QACL,IAAI,CAAC,EAAE,WAAW,CAAC;KACnB,CAAC;CACF,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;AAEhC,qBACa,SAAS,CACrB,CAAC,EACD,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CACtE,SAAQ,OAAO,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC;IAEvE,GAAG,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;IAG1B,SAAS,EAAE,OAAO,CAAC;IAGnB,OAAO,CAAC,MAAM,CAAsB;IAEpC,OAAO,CAAC,MAAM,CAAmB;IAEjC,OAAO,CAAC,iBAAiB,CAAC,CAAgB;IAC1C,OAAO,CAAC,eAAe,CAAe;IACtC,OAAO,CAAC,UAAU,CAA0C;IAE5D,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;gBAElE,UAAU,CAAC,EAAE;QACxB,EAAE,CAAC,EAAE,UAAU,CAAC;QAChB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,KAAK,CAAC,EAAE,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;KAC5B;IAQD,IAAI,KAAK,IAAI,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAE/B;IAEK,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC;IA8DhC,OAAO;YAIC,aAAa;IAOrB,SAAS,CACd,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,EACvB,SAAS,CAAC,EAAE;QAAE,QAAQ,EAAE,CAAC,CAAC;QAAC,SAAS,EAAE,YAAY,CAAA;KAAE,GAClD,OAAO,CAAC,OAAO,CAAC;IAuDb,UAAU,CACf,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,EACvB,SAAS,CAAC,EAAE;QAAE,QAAQ,EAAE,CAAC,CAAC;QAAC,SAAS,EAAE,YAAY,CAAA;KAAE,GAClD,OAAO,CAAC,YAAY,GAAG,eAAe,GAAG,KAAK,CAAC;IAyGrC,GAAG,CACf,GAAG,EAAE,CAAC,EACN,OAAO,CAAC,EAAE,mBAAmB,CAAC,SAAS,CAAC,GAAG;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE;;;;IA+C1D,GAAG,CACR,EAAE,EAAE,YAAY,CAAC,OAAO,EACxB,OAAO,CAAC,EAAE,mBAAmB,CAAC,SAAS,CAAC;;;;IA6BnC,aAAa,CAClB,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,EACzB,SAAS,CAAC,EAAE;QAAE,QAAQ,EAAE,CAAC,CAAC;QAAC,SAAS,EAAE,YAAY,CAAA;KAAE,GAClD,OAAO,CAAC,IAAI,CAAC;CAoIhB"}

package/{lib/esm/document-store.js → dist/src/program.js}

@@ -7,17 +7,17 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
-import { BorshError, deserialize, field, serialize, variant } from "@dao-xyz/borsh";
-import { Entry, EntryType } from "@peerbit/log";
-import { Program } from "@peerbit/program";
+import { BorshError, field, serialize, variant, } from "@dao-xyz/borsh";
+import { CustomEvent } from "@libp2p/interface";
 import { AccessError, DecryptedThing } from "@peerbit/crypto";
+import * as documentsTypes from "@peerbit/document-interface";
+import * as indexerTypes from "@peerbit/indexer-interface";
+import { Entry, EntryType, } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
-import { CustomEvent } from "@libp2p/interface";
-import { Replicator, SharedLog } from "@peerbit/shared-log";
-import { BORSH_ENCODING_OPERATION, DeleteOperation, DocumentIndex, PutOperation, MAX_DOCUMENT_SIZE } from "./document-index.js";
-import { asString, checkKeyable } from "./utils.js";
-import { Context } from "./query.js";
-export { MAX_DOCUMENT_SIZE };
+import { Program } from "@peerbit/program";
+import { SharedLog, } from "@peerbit/shared-log";
+import { MAX_BATCH_SIZE } from "./constants.js";
+import { BORSH_ENCODING_OPERATION, DeleteOperation, DocumentIndex, Operation, PutOperation, } from "./search.js";
 const logger = loggerFn({ module: "document" });
 export class OperationError extends Error {
     constructor(message) {
@@ -31,6 +31,7 @@ let Documents = class Documents extends Program {
     _clazz;
     _optionCanPerform;
     _manuallySynced;
+    idResolver;
     canOpen;
     constructor(properties) {
         super();
@@ -52,13 +53,19 @@ let Documents = class Documents extends Program {
         }
         this._optionCanPerform = options.canPerform;
         this._manuallySynced = new Set();
+        const idProperty = options.index?.idProperty || "id";
+        const idResolver = options.id ||
+            (typeof idProperty === "string"
+                ? (obj) => obj[idProperty]
+                : (obj) => indexerTypes.extractFieldValue(obj, idProperty));
+        this.idResolver = idResolver;
         await this._index.open({
-            type: this._clazz,
             log: this.log,
             canRead: options?.index?.canRead,
             canSearch: options.index?.canSearch,
-            fields: options.index?.fields || ((obj) => obj),
-            indexBy: options.index?.key,
+            documentType: this._clazz,
+            transform: options.index,
+            indexBy: idProperty,
             sync: async (result) => {
                 // here we arrive for all the results we want to persist.
                 // we we need to do here is
@@ -70,7 +77,7 @@ let Documents = class Documents extends Program {
                 }
                 return this.log.log.join(heads);
             },
-            dbType: this.constructor
+            dbType: this.constructor,
         });
         await this.log.open({
             encoding: BORSH_ENCODING_OPERATION,
@@ -78,13 +85,13 @@ let Documents = class Documents extends Program {
             canAppend: this.canAppend.bind(this),
             onChange: this.handleChanges.bind(this),
             trim: options?.log?.trim,
-            role: options?.role,
+            replicate: options?.replicate,
             replicas: options?.replicas,
             sync: (entry) => {
                 // here we arrive when ever a insertion/pruning behaviour processes an entry
                 // returning true means that it should persist
                 return this._manuallySynced.has(entry.gid);
-            }
+            },
         });
     }
     async recover() {
@@ -96,26 +103,43 @@ let Documents = class Documents extends Program {
                 (await Entry.fromMultihash(this.log.log.blocks, history))
             : history;
     }
-    async updateRole(role) {
-        await this.log.updateRole(role);
-    }
-    get role() {
-        return this.log.role;
-    }
-    async canAppend(entry) {
-        const l0 = await this._canAppend(entry);
+    async canAppend(entry, reference) {
+        const l0 = await this._canAppend(entry, reference);
         if (!l0) {
             return false;
         }
         try {
-            const payload = await entry.getPayloadValue();
-            if (payload instanceof PutOperation) {
-                payload.getValue(this.index.valueEncoding); // Decode they value so callbacks can jsut do .value
+            let operation = l0;
+            let document = reference?.document;
+            if (!document) {
+                if (l0 instanceof PutOperation) {
+                    document = this._index.valueEncoding.decoder(l0.data);
+                    if (!document) {
+                        return false;
+                    }
+                }
+                else if (l0 instanceof DeleteOperation) {
+                    // Nothing to do here by default
+                    // checking if the document exists is not necessary
+                    // since it might already be deleted
+                }
+                else {
+                    throw new Error("Unsupported operation");
+                }
             }
             if (this._optionCanPerform) {
-                if (!(await this._optionCanPerform(payload, {
-                    entry
-                }))) {
+                if (!(await this._optionCanPerform(operation instanceof PutOperation
+                    ? {
+                        type: "put",
+                        value: document,
+                        operation,
+                        entry: entry,
+                    }
+                    : {
+                        type: "delete",
+                        operation,
+                        entry: entry,
+                    }))) {
                     return false;
                 }
             }
@@ -129,7 +153,7 @@ let Documents = class Documents extends Program {
         }
         return true;
     }
-    async _canAppend(entry) {
+    async _canAppend(entry, reference) {
         const resolve = async (history) => {
             return typeof history === "string"
                 ? this.log.log.get(history) ||
@@ -153,18 +177,22 @@ let Documents = class Documents extends Program {
         try {
             entry.init({
                 encoding: this.log.log.encoding,
-                keychain: this.node.services.keychain
+                keychain: this.node.services.keychain,
             });
-            const operation = entry._payload instanceof DecryptedThing
+            const operation = reference?.operation || entry._payload instanceof DecryptedThing
                 ? entry.payload.getValue(entry.encoding)
                 : await entry.getPayloadValue();
             if (operation instanceof PutOperation) {
                 // check nexts
                 const putOperation = operation;
-                const key = this._index.indexByResolver(putOperation.getValue(this.index.valueEncoding));
-                checkKeyable(key);
-                const existingDocument = this.index.index.get(asString(key));
-                if (existingDocument) {
+                let value = reference?.document ??
+                    this.index.valueEncoding.decoder(putOperation.data);
+                const keyValue = this.idResolver(value);
+                const key = indexerTypes.toId(keyValue);
+                const existingDocument = (await this.index.getDetailed(key))?.[0]
+                    ?.results[0];
+                if (existingDocument && existingDocument.context.head !== entry.hash) {
+                    //  econd condition can false if we reset the operation log, while not  resetting the index. For example when doing .recover
                     if (this.immutable) {
                         //Key already exist and this instance Documents can note overrite/edit'
                         return false;
@@ -177,7 +205,8 @@ let Documents = class Documents extends Program {
                         logger.error("Failed to find Document from head");
                         return false;
                     }
-                    return pointsToHistory(doc);
+                    const referenceHistoryCorrectly = await pointsToHistory(doc);
+                    return referenceHistoryCorrectly ? putOperation : false;
                 }
                 else {
                     if (entry.next.length !== 0) {
@@ -189,18 +218,26 @@ let Documents = class Documents extends Program {
                 if (entry.next.length !== 1) {
                     return false;
                 }
-                const existingDocument = this._index.index.get(operation.key);
+                const existingDocument = (await this.index.getDetailed(operation.key))?.[0].results[0];
                 if (!existingDocument) {
                     // already deleted
-                    return true; // assume ok
+                    return operation; // assume ok
                 }
                 let doc = await this.log.log.get(existingDocument.context.head);
                 if (!doc) {
                     logger.error("Failed to find Document from head");
                     return false;
                 }
-                return pointsToHistory(doc); // references the existing document
+                if (await pointsToHistory(doc)) {
+                    // references the existing document
+                    return operation;
+                }
+                return false;
             }
+            else {
+                throw new Error("Unsupported operation");
+            }
+            return operation;
         }
         catch (error) {
             if (error instanceof AccessError) {
@@ -212,144 +249,145 @@ let Documents = class Documents extends Program {
             }
             throw error;
         }
-        return true;
     }
     async put(doc, options) {
-        const key = this._index.indexByResolver(doc);
-        checkKeyable(key);
+        const keyValue = this.idResolver(doc);
+        // type check the key
+        indexerTypes.checkId(keyValue);
         const ser = serialize(doc);
-        if (ser.length > MAX_DOCUMENT_SIZE) {
-            throw new Error(`Document is too large (${ser.length * 1e-6}) mb). Needs to be less than ${MAX_DOCUMENT_SIZE * 1e-6} mb`);
+        if (ser.length > MAX_BATCH_SIZE) {
+            throw new Error(`Document is too large (${ser.length * 1e-6}) mb). Needs to be less than ${MAX_BATCH_SIZE * 1e-6} mb`);
         }
         const existingDocument = options?.unique
             ? undefined
-            : (await this._index.getDetailed(key, {
+            : (await this._index.getDetailed(keyValue, {
                 local: true,
-                remote: { sync: true } // only query remote if we know they exist
+                remote: { sync: true }, // only query remote if we know they exist
             }))?.[0]?.results[0];
-        return this.log.append(new PutOperation({
-            key: asString(key),
+        const operation = new PutOperation({
             data: ser,
-            value: doc
-        }), {
+        });
+        const appended = await this.log.append(operation, {
             ...options,
             meta: {
                 next: existingDocument
                     ? [await this._resolveEntry(existingDocument.context.head)]
                     : [],
-                ...options?.meta
-            } //
+                ...options?.meta,
+            },
+            canAppend: (entry) => {
+                return this.canAppend(entry, { document: doc, operation });
+            },
+            onChange: (change) => {
+                return this.handleChanges(change, { document: doc, operation });
+            },
         });
+        return appended;
     }
-    async del(key, options) {
+    async del(id, options) {
+        const key = indexerTypes.toId(id);
         const existing = (await this._index.getDetailed(key, {
             local: true,
-            remote: { sync: true }
+            remote: { sync: true },
         }))?.[0]?.results[0];
         if (!existing) {
-            throw new Error(`No entry with key '${key}' in the database`);
+            throw new Error(`No entry with key '${key.primitive}' in the database`);
         }
         return this.log.append(new DeleteOperation({
-            key: asString(key)
+            key,
         }), {
             ...options,
             meta: {
                 next: [await this._resolveEntry(existing.context.head)],
                 type: EntryType.CUT,
-                ...options?.meta
-            }
-        } //
-        );
+                ...options?.meta,
+            },
+        });
     }
-    async handleChanges(change) {
-        const removed = [...(change.removed || [])];
+    async handleChanges(change, reference) {
+        const isAppendOperation = change?.added.length === 1 ? !!change.added[0] : false;
         const removedSet = new Map();
-        for (const r of removed) {
+        for (const r of change.removed) {
             removedSet.set(r.hash, r);
         }
-        const entries = [...change.added, ...(removed || [])]
-            .sort(this.log.log.sortFn)
-            .reverse(); // sort so we get newest to oldest
+        const sortedEntries = [
+            ...change.added,
+            ...((await Promise.all(change.removed.map((x) => x instanceof Entry ? x : this.log.log.entryIndex.get(x.hash)))) || []),
+        ]; // TODO assert sorting
+        /*
+                const sortedEntries = [...change.added, ...(removed || [])]
+                    .sort(this.log.log.sortFn)
+                    .reverse(); // sort so we get newest to oldest */
         // There might be a case where change.added and change.removed contains the same document id. Usaully because you use the "trim" option
         // in combination with inserting the same document. To mitigate this, we loop through the changes and modify the behaviour for this
-        let visited = new Map();
-        for (const item of entries) {
-            const payload = item._payload instanceof DecryptedThing
-                ? item.payload.getValue(item.encoding)
-                : await item.getPayloadValue();
-            let itemKey;
-            if (payload instanceof PutOperation ||
-                payload instanceof DeleteOperation) {
-                itemKey = payload.key;
-            }
-            else {
-                throw new Error("Unsupported operation type");
-            }
-            let arr = visited.get(itemKey);
-            if (!arr) {
-                arr = [];
-                visited.set(itemKey, arr);
-            }
-            arr.push(item);
-        }
         let documentsChanged = {
             added: [],
-            removed: []
+            removed: [],
         };
-        for (const [_key, entries] of visited) {
+        let modified = new Set();
+        for (const item of sortedEntries) {
+            if (!item)
+                continue;
             try {
-                const item = entries[0];
                 const payload = item._payload instanceof DecryptedThing
                     ? item.payload.getValue(item.encoding)
                     : await item.getPayloadValue();
                 if (payload instanceof PutOperation && !removedSet.has(item.hash)) {
-                    const key = payload.key;
-                    let value = this.deserializeOrPass(payload);
+                    let value = (isAppendOperation &&
+                        reference?.operation === payload &&
+                        reference?.document) ||
+                        this.index.valueEncoding.decoder(payload.data);
+                    // get index key from value
+                    const keyObject = this.idResolver(value);
+                    const key = indexerTypes.toId(keyObject);
+                    // document is already updated with more recent entry
+                    if (modified.has(key.primitive)) {
+                        continue;
+                    }
                     // Program specific
                     if (value instanceof Program) {
                         // if replicator, then open
                         if ((await this.canOpen(value, item)) &&
-                            this.log.role instanceof Replicator &&
-                            (await this.log.replicator(item)) // TODO types, throw runtime error if replicator is not provided
+                            (await this.log.isReplicator(item)) // TODO types, throw runtime error if replicator is not provided
                         ) {
                             value = (await this.node.open(value, {
                                 parent: this,
-                                existing: "reuse"
+                                existing: "reuse",
                             })); // TODO types
                         }
                     }
                     documentsChanged.added.push(value);
-                    const context = new Context({
-                        created: this._index.index.get(key)?.context.created ||
-                            item.meta.clock.timestamp.wallTime,
-                        modified: item.meta.clock.timestamp.wallTime,
-                        head: item.hash,
-                        gid: item.gid
-                    });
-                    const valueToIndex = this._index.toIndex(value, context);
-                    this._index.index.set(key, {
-                        key: payload.key,
-                        value: isPromise(valueToIndex) ? await valueToIndex : valueToIndex,
-                        context,
-                        reference: valueToIndex === value || value instanceof Program
-                            ? { value, last: payload }
-                            : undefined
-                    });
+                    await this._index.put(value, item, key);
+                    modified.add(key.primitive);
                 }
                 else if ((payload instanceof DeleteOperation && !removedSet.has(item.hash)) ||
                     payload instanceof PutOperation ||
                     removedSet.has(item.hash)) {
                     this._manuallySynced.delete(item.gid);
-                    const key = payload.key;
-                    if (!this.index.index.has(key)) {
-                        continue;
-                    }
                     let value;
+                    let key;
                     if (payload instanceof PutOperation) {
-                        value = this.deserializeOrPass(payload);
+                        value = this.index.valueEncoding.decoder(payload.data);
+                        key = indexerTypes.toId(this.idResolver(value));
+                        // document is already updated with more recent entry
+                        if (modified.has(key.primitive)) {
+                            continue;
+                        }
                     }
                     else if (payload instanceof DeleteOperation) {
-                        value = await this.getDocumentFromEntry(entries[1]);
+                        key = payload.key;
+                        // document is already updated with more recent entry
+                        if (modified.has(key.primitive)) {
+                            continue;
+                        }
+                        const document = await this._index.get(key, {
+                            local: true,
+                            remote: false,
+                        });
+                        if (!document) {
+                            continue;
+                        }
+                        value = document;
                     }
                     else {
                         throw new Error("Unexpected");
@@ -359,10 +397,12 @@ let Documents = class Documents extends Program {
                         await value.drop(this);
                     }
                     // update index
-                    this._index.index.delete(key);
+                    await this._index.del(key);
+                    modified.add(key.primitive);
                 }
                 else {
                     // Unknown operation
+                    throw new OperationError("Unknown operation");
                 }
             }
             catch (error) {
@@ -374,22 +414,6 @@ let Documents = class Documents extends Program {
         }
         this.events.dispatchEvent(new CustomEvent("change", { detail: documentsChanged }));
     }
-    async getDocumentFromEntry(entry) {
-        const payloadValue = await entry.getPayloadValue();
-        if (payloadValue instanceof PutOperation) {
-            return payloadValue.getValue(this.index.valueEncoding);
-        }
-        throw new Error("Unexpected");
-    }
-    deserializeOrPass(value) {
-        if (value._value) {
-            return value._value;
-        }
-        else {
-            value._value = deserialize(value.data, this.index.type);
-            return value._value;
-        }
-    }
 };
 __decorate([
     field({ type: SharedLog }),
@@ -408,7 +432,4 @@ Documents = __decorate([
     __metadata("design:paramtypes", [Object])
 ], Documents);
 export { Documents };
-function isPromise(value) {
-    return Boolean(value && typeof value.then === "function");
-}
-//# sourceMappingURL=document-store.js.map
+//# sourceMappingURL=program.js.map