@xyo-network/archivist-storage 3.5.2 → 3.6.0-rc.10

package/dist/neutral/StorageArchivist.d.ts

@@ -1,13 +1,10 @@
-import type { Hash } from '@xylabs/hex';
+import type { Hash, Hex } from '@xylabs/hex';
 import type { Promisable, PromisableArray } from '@xylabs/promise';
 import { AbstractArchivist } from '@xyo-network/archivist-abstract';
 import type { ArchivistConfig, ArchivistInstance, ArchivistModuleEventData, ArchivistNextOptions, ArchivistParams } from '@xyo-network/archivist-model';
 import type { BoundWitness } from '@xyo-network/boundwitness-model';
 import type { AnyConfigSchema } from '@xyo-network/module-model';
-import type { Payload, PayloadWithMeta, Schema, WithMeta } from '@xyo-network/payload-model';
-type WithStorageMeta<T extends Payload = Payload> = WithMeta<T> & {
-    _timestamp: number;
-};
+import type { Payload, Schema, WithStorageMeta } from '@xyo-network/payload-model';
 export type StorageArchivistConfigSchema = 'network.xyo.archivist.storage.config';
 export declare const StorageArchivistConfigSchema: StorageArchivistConfigSchema;
 export type StorageArchivistConfig = ArchivistConfig<{
@@ -30,17 +27,14 @@ export declare class StorageArchivist<TParams extends StorageArchivistParams = S
     get type(): "local" | "session" | "page";
     private get privateStorage();
     private get storage();
-    protected allHandler(): PromisableArray<PayloadWithMeta>;
+    protected allHandler(): PromisableArray<WithStorageMeta<Payload>>;
     protected clearHandler(): void | Promise<void>;
-    protected commitHandler(): Promise<WithMeta<BoundWitness>[]>;
+    protected commitHandler(): Promise<BoundWitness[]>;
     protected deleteHandler(hashes: Hash[]): Promise<Hash[]>;
-    protected getFromOffset(order?: 'asc' | 'desc', limit?: number, offset?: Hash): WithStorageMeta[];
-    protected getHandler(hashes: string[]): Promisable<PayloadWithMeta[]>;
-    protected insertHandler(payloads: Payload[]): Promise<PayloadWithMeta[]>;
-    protected nextHandler(options?: ArchivistNextOptions): Promisable<PayloadWithMeta[]>;
+    protected getFromCursor(order?: 'asc' | 'desc', limit?: number, cursor?: Hex): WithStorageMeta[];
+    protected getHandler(hashes: string[]): Promisable<WithStorageMeta<Payload>[]>;
+    protected insertHandler(payloads: Payload[]): Promise<WithStorageMeta<Payload>[]>;
+    protected nextHandler(options?: ArchivistNextOptions): Promisable<WithStorageMeta<Payload>[]>;
     protected startHandler(): Promise<boolean>;
-    private addStorageMeta;
-    private removeStorageMeta;
 }
-export {};
 //# sourceMappingURL=StorageArchivist.d.ts.map

package/dist/neutral/StorageArchivist.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"StorageArchivist.d.ts","sourceRoot":"","sources":["../../src/StorageArchivist.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAElE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,KAAK,EACV,eAAe,EAEf,iBAAiB,EACjB,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EAChB,MAAM,8BAA8B,CAAA;AAQrC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAEhE,OAAO,KAAK,EACV,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,EAC3C,MAAM,4BAA4B,CAAA;AAMnC,KAAK,eAAe,CAAC,CAAC,SAAS,OAAO,GAAG,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,GAAG;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,CAAA;AAExF,MAAM,MAAM,4BAA4B,GAAG,sCAAsC,CAAA;AACjF,eAAO,MAAM,4BAA4B,EAAE,4BAAqE,CAAA;AAEhH,MAAM,MAAM,sBAAsB,GAAG,eAAe,CAAC;IACnD,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,4BAA4B,CAAA;IACpC,IAAI,CAAC,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAA;CACpC,CAAC,CAAA;AAEF,MAAM,MAAM,sBAAsB,GAAG,eAAe,CAAC,eAAe,CAAC,sBAAsB,CAAC,CAAC,CAAA;AAC7F,qBAAa,gBAAgB,CAC3B,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,UAAU,SAAS,wBAAwB,GAAG,wBAAwB,CAEtE,SAAQ,iBAAiB,CAAC,OAAO,EAAE,UAAU,CAC7C,YAAW,iBAAiB;IAC5B,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAyD;IACzG,gBAAyB,mBAAmB,EAAE,MAAM,CAA+B;IAEnF,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,QAAQ,CAAuB;IAEvC,IAAI,UAAU,WAEb;IAED,IAAI,YAAY,WAEf;IAED,IAAI,SAAS,WAEZ;IAED,IAAa,OAAO,IAAI,MAAM,EAAE,CAS/B;IAED,IAAI,IAAI,iCAEP;IAGD,OAAO,KAAK,cAAc,GAGzB;IAGD,OAAO,KAAK,OAAO,GAGlB;cAEkB,UAAU,IAAI,eAAe,CAAC,eAAe,CAAC;cAiB9C,YAAY,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;cAM9B,aAAa,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;cAgBlD,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAWvE,SAAS,CAAC,aAAa,CACrB,KAAK,GAAE,KAAK,GAAG,MAAc,EAC7B,KAAK,GAAE,MAAW,EAClB,MAAM,CAAC,EAAE,IAAI,GACZ,eAAe,EAAE;cAyBD,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,eAAe,EAAE,CAAC;cAiBrD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;cAcpE,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,UAAU,CAAC,eAAe,EAAE,CAAC;cAOpE,YAAY;IAKrC,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,iBAAiB;CAK1B"}
+{"version":3,"file":"StorageArchivist.d.ts","sourceRoot":"","sources":["../../src/StorageArchivist.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAElE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,KAAK,EACV,eAAe,EAEf,iBAAiB,EACjB,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EAChB,MAAM,8BAA8B,CAAA;AAQrC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAEhE,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAAE,eAAe,EACjC,MAAM,4BAA4B,CAAA;AAMnC,MAAM,MAAM,4BAA4B,GAAG,sCAAsC,CAAA;AACjF,eAAO,MAAM,4BAA4B,EAAE,4BAAqE,CAAA;AAEhH,MAAM,MAAM,sBAAsB,GAAG,eAAe,CAAC;IACnD,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,4BAA4B,CAAA;IACpC,IAAI,CAAC,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAA;CACpC,CAAC,CAAA;AAEF,MAAM,MAAM,sBAAsB,GAAG,eAAe,CAAC,eAAe,CAAC,sBAAsB,CAAC,CAAC,CAAA;AAC7F,qBAAa,gBAAgB,CAC3B,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,UAAU,SAAS,wBAAwB,GAAG,wBAAwB,CAEtE,SAAQ,iBAAiB,CAAC,OAAO,EAAE,UAAU,CAC7C,YAAW,iBAAiB;IAC5B,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAyD;IACzG,gBAAyB,mBAAmB,EAAE,MAAM,CAA+B;IAEnF,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,QAAQ,CAAuB;IAEvC,IAAI,UAAU,WAEb;IAED,IAAI,YAAY,WAEf;IAED,IAAI,SAAS,WAEZ;IAED,IAAa,OAAO,IAAI,MAAM,EAAE,CAS/B;IAED,IAAI,IAAI,iCAEP;IAGD,OAAO,KAAK,cAAc,GAGzB;IAGD,OAAO,KAAK,OAAO,GAGlB;cAEkB,UAAU,IAAI,eAAe,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;cAiBvD,YAAY,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;cAM9B,aAAa,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;cAgBxC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAWvE,SAAS,CAAC,aAAa,CACrB,KAAK,GAAE,KAAK,GAAG,MAAc,EAC7B,KAAK,GAAE,MAAW,EAClB,MAAM,CAAC,EAAE,GAAG,GACX,eAAe,EAAE;cAcD,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAiB9D,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAY7E,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAO7E,YAAY;CAItC"}

package/dist/neutral/index.mjs

@@ -58,13 +58,13 @@ var StorageArchivist = class extends AbstractArchivist {
     const found = /* @__PURE__ */ new Set();
     this.logger?.log(`this.storage.length: ${this.storage.length}`);
     return Object.entries(this.storage.getAll()).map(([, value]) => value).filter((payload) => {
-      if (found.has(payload.$hash)) {
+      if (found.has(payload._dataHash)) {
         return false;
       } else {
-        found.add(payload.$hash);
+        found.add(payload._dataHash);
         return true;
       }
-    }).sort((a, b) => a._timestamp - b._timestamp).map((payload) => this.removeStorageMeta(payload));
+    }).sort(PayloadBuilder.compareStorageMeta).map((payload) => PayloadBuilder.omitStorageMeta(payload));
   }
   clearHandler() {
     this.logger?.log(`this.storage.length: ${this.storage.length}`);
@@ -93,24 +93,14 @@ var StorageArchivist = class extends AbstractArchivist {
       return hash;
     }))).filter(exists);
   }
-  getFromOffset(order = "asc", limit = 10, offset) {
-    const offsetHash = offset ? this.storage.get(offset)?.$hash : void 0;
-    const found = /* @__PURE__ */ new Set();
-    const payloads = Object.entries(this.storage.getAll()).map(([, value]) => value).filter((payload) => {
-      if (found.has(payload.$hash)) {
-        return false;
-      } else {
-        found.add(payload.$hash);
-        return true;
-      }
-    }).sort((a, b) => {
-      return order === "asc" ? a._timestamp - b._timestamp : b._timestamp - a._timestamp;
+  getFromCursor(order = "asc", limit = 10, cursor) {
+    const all = Object.values(this.storage.getAll());
+    const payloads = all.map((value) => value).sort((a, b) => {
+      return order === "asc" ? PayloadBuilder.compareStorageMeta(a, b) : PayloadBuilder.compareStorageMeta(b, a);
     });
-    if (offsetHash) {
-      const index = payloads.findIndex((payload) => payload.$hash === offsetHash);
-      if (index !== -1) {
-        return payloads.slice(index + 1, index + 1 + limit);
-      }
+    const index = payloads.findIndex((payload) => payload._sequence === cursor);
+    if (index !== -1) {
+      return payloads.slice(index + 1, index + 1 + limit);
     }
     return payloads.slice(0, limit);
   }
@@ -119,45 +109,32 @@ var StorageArchivist = class extends AbstractArchivist {
     return hashes.map((hash) => {
       return this.storage.get(hash);
     }).filter(exists).filter((payload) => {
-      if (found.has(payload.$hash)) {
+      if (found.has(payload._dataHash)) {
         return false;
       } else {
-        found.add(payload.$hash);
+        found.add(payload._dataHash);
         return true;
       }
-    }).map((payload) => this.removeStorageMeta(payload));
+    }).map((payload) => PayloadBuilder.omitStorageMeta(payload));
   }
   async insertHandler(payloads) {
-    let timestamp = Date.now();
-    const pairs = await PayloadBuilder.hashPairs(payloads);
-    return pairs.map(([payload, hash]) => {
-      const storagePayload = this.addStorageMeta(payload, timestamp++);
+    const storagePayloads = (await PayloadBuilder.addStorageMeta(payloads)).sort(PayloadBuilder.compareStorageMeta);
+    return await Promise.all(storagePayloads.map((storagePayload) => {
       const value = JSON.stringify(storagePayload);
-      console.log("insert.storagePayloads:", storagePayload);
-      assertEx(value.length < this.maxEntrySize, () => `Payload too large [${hash}, ${value.length}]`);
-      this.storage.set(hash, storagePayload);
-      this.storage.set(payload.$hash, storagePayload);
-      return payload;
-    });
+      assertEx(value.length < this.maxEntrySize, () => `Payload too large [${storagePayload._hash}, ${value.length}]`);
+      this.storage.set(storagePayload._hash, storagePayload);
+      this.storage.set(storagePayload._dataHash, storagePayload);
+      return storagePayload;
+    }));
   }
   nextHandler(options) {
-    const { limit, offset, order } = options ?? {};
-    return this.getFromOffset(order, limit ?? 10, offset);
+    const { limit, cursor, order } = options ?? {};
+    return this.getFromCursor(order, limit ?? 10, cursor);
   }
   async startHandler() {
     await super.startHandler();
     return true;
   }
-  addStorageMeta(payload, _timestamp) {
-    return {
-      ...payload,
-      _timestamp
-    };
-  }
-  removeStorageMeta(payload) {
-    const { _timestamp, ...rest } = payload;
-    return rest;
-  }
 };
 export {
   StorageArchivist,

package/dist/neutral/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/StorageArchivist.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport type { Promisable, PromisableArray } from '@xylabs/promise'\nimport { fulfilled } from '@xylabs/promise'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport type {\n  ArchivistConfig,\n  ArchivistInsertQuery,\n  ArchivistInstance,\n  ArchivistModuleEventData,\n  ArchivistNextOptions,\n  ArchivistParams,\n} from '@xyo-network/archivist-model'\nimport {\n  ArchivistAllQuerySchema,\n  ArchivistClearQuerySchema,\n  ArchivistCommitQuerySchema,\n  ArchivistDeleteQuerySchema,\n  ArchivistInsertQuerySchema,\n} from '@xyo-network/archivist-model'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport type { AnyConfigSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, PayloadWithMeta, Schema, WithMeta,\n} from '@xyo-network/payload-model'\nimport type { StoreBase, StoreType } from 'store2'\nimport store from 'store2'\n\nconst storeTypes = store as unknown as StoreType\n\ntype WithStorageMeta<T extends Payload = Payload> = WithMeta<T> & { _timestamp: number }\n\nexport type StorageArchivistConfigSchema = 'network.xyo.archivist.storage.config'\nexport const StorageArchivistConfigSchema: StorageArchivistConfigSchema = 'network.xyo.archivist.storage.config'\n\nexport type StorageArchivistConfig = ArchivistConfig<{\n  maxEntries?: number\n  maxEntrySize?: number\n  namespace?: string\n  schema: StorageArchivistConfigSchema\n  type?: 'local' | 'session' | 'page'\n}>\n\nexport type StorageArchivistParams = ArchivistParams<AnyConfigSchema<StorageArchivistConfig>>\nexport class StorageArchivist<\n  TParams extends StorageArchivistParams = StorageArchivistParams,\n  TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n>\n  extends AbstractArchivist<TParams, TEventData>\n  implements ArchivistInstance {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, StorageArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = StorageArchivistConfigSchema\n\n  private _privateStorage: StoreBase | undefined\n  private _storage: StoreBase | undefined\n\n  get maxEntries() {\n    return this.config?.maxEntries ?? 1000\n  }\n\n  get maxEntrySize() {\n    return this.config?.maxEntrySize ?? 16_000\n  }\n\n  get namespace() {\n    return this.config?.namespace ?? 'xyo-archivist'\n  }\n\n  override get queries(): string[] {\n    return [\n      ArchivistAllQuerySchema,\n      ArchivistDeleteQuerySchema,\n      ArchivistClearQuerySchema,\n      ArchivistInsertQuerySchema,\n      ArchivistCommitQuerySchema,\n      ...super.queries,\n    ]\n  }\n\n  get type() {\n    return this.config?.type ?? 'local'\n  }\n\n  /* This has to be a getter so that it can access it during construction */\n  private get privateStorage(): StoreBase {\n    this._privateStorage = this._storage ?? storeTypes[this.type].namespace(`${this.namespace}|private`)\n    return this._privateStorage\n  }\n\n  /* This has to be a getter so that it can access it during construction */\n  private get storage(): StoreBase {\n    this._storage = this._storage ?? storeTypes[this.type].namespace(this.namespace)\n    return this._storage\n  }\n\n  protected override allHandler(): PromisableArray<PayloadWithMeta> {\n    const found = new Set<string>()\n    this.logger?.log(`this.storage.length: ${this.storage.length}`)\n    return Object.entries(this.storage.getAll())\n      .map(([, value]) => value)\n      .filter((payload) => {\n        if (found.has(payload.$hash)) {\n          return false\n        } else {\n          found.add(payload.$hash)\n          return true\n        }\n      })\n      .sort((a, b) => a._timestamp - b._timestamp)\n      .map(payload => this.removeStorageMeta(payload))\n  }\n\n  protected override clearHandler(): void | Promise<void> {\n    this.logger?.log(`this.storage.length: ${this.storage.length}`)\n    this.storage.clear()\n    return this.emit('cleared', { mod: this })\n  }\n\n  protected override async commitHandler(): Promise<WithMeta<BoundWitness>[]> {\n    this.logger?.log(`this.storage.length: ${this.storage.length}`)\n    const payloads = await this.all()\n    assertEx(payloads.length > 0, () => 'Nothing to commit')\n    const settled = (await Promise.allSettled(\n      Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n        const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n        const query = await this.bindQuery(queryPayload, payloads)\n        return (await parent?.query(query[0], query[1]))?.[0]\n      }),\n    )).filter(exists)\n    // TODO - rather than clear, delete the payloads that come back as successfully inserted\n    await this.clear()\n    return (settled.filter(fulfilled).map(result => result.value)).filter(exists)\n  }\n\n  protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {\n    return (\n      await Promise.all(\n        hashes.map((hash) => {\n          this.storage.remove(hash)\n          return hash\n        }),\n      )\n    ).filter(exists)\n  }\n\n  protected getFromOffset(\n    order: 'asc' | 'desc' = 'asc',\n    limit: number = 10,\n    offset?: Hash,\n  ): WithStorageMeta[] {\n    const offsetHash = offset ? (this.storage.get(offset) as PayloadWithMeta | undefined)?.$hash : undefined\n    const found = new Set<string>()\n    const payloads: WithStorageMeta[] = Object.entries(this.storage.getAll())\n      .map(([, value]) => value)\n      .filter((payload) => {\n        if (found.has(payload.$hash)) {\n          return false\n        } else {\n          found.add(payload.$hash)\n          return true\n        }\n      })\n      .sort((a, b) => {\n        return order === 'asc' ? a._timestamp - b._timestamp : b._timestamp - a._timestamp\n      })\n    if (offsetHash) {\n      const index = payloads.findIndex(payload => payload.$hash === offsetHash)\n      if (index !== -1) {\n        return payloads.slice(index + 1, index + 1 + limit)\n      }\n    }\n    return payloads.slice(0, limit)\n  }\n\n  protected override getHandler(hashes: string[]): Promisable<PayloadWithMeta[]> {\n    const found = new Set<string>()\n    return (\n      hashes.map((hash) => {\n        return this.storage.get(hash)\n      })\n    ).filter(exists)\n      .filter((payload) => {\n        if (found.has(payload.$hash)) {\n          return false\n        } else {\n          found.add(payload.$hash)\n          return true\n        }\n      }).map(payload => this.removeStorageMeta(payload))\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<PayloadWithMeta[]> {\n    let timestamp = Date.now()\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    return pairs.map(([payload, hash]) => {\n      const storagePayload = this.addStorageMeta(payload, timestamp++)\n      const value = JSON.stringify(storagePayload)\n      console.log('insert.storagePayloads:', storagePayload)\n      assertEx(value.length < this.maxEntrySize, () => `Payload too large [${hash}, ${value.length}]`)\n      this.storage.set(hash, storagePayload)\n      this.storage.set(payload.$hash, storagePayload)\n      return payload\n    })\n  }\n\n  protected override nextHandler(options?: ArchivistNextOptions): Promisable<PayloadWithMeta[]> {\n    const {\n      limit, offset, order,\n    } = options ?? {}\n    return this.getFromOffset(order, limit ?? 10, offset)\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    return true\n  }\n\n  private addStorageMeta<T extends Payload = Payload>(payload: WithMeta<T>, _timestamp: number): WithStorageMeta<T> {\n    return { ...payload, _timestamp }\n  }\n\n  private removeStorageMeta<T extends Payload = Payload>(payload: WithStorageMeta<T>): WithMeta<T> {\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { _timestamp, ...rest } = payload\n    return rest as WithMeta<T>\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,iBAAiB;AAC1B,SAASC,yBAAyB;AASlC,SACEC,yBACAC,2BACAC,4BACAC,4BACAC,kCACK;AAGP,SAASC,sBAAsB;AAK/B,OAAOC,WAAW;AAElB,IAAMC,aAAaC;AAKZ,IAAMC,+BAA6D;AAWnE,IAAMC,mBAAN,cAIGC,kBAAAA;EAlDV,OAkDUA;;;EAER,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE/CK;EACAC;EAER,IAAIC,aAAa;AACf,WAAO,KAAKC,QAAQD,cAAc;EACpC;EAEA,IAAIE,eAAe;AACjB,WAAO,KAAKD,QAAQC,gBAAgB;EACtC;EAEA,IAAIC,YAAY;AACd,WAAO,KAAKF,QAAQE,aAAa;EACnC;EAEA,IAAaC,UAAoB;AAC/B,WAAO;MACLC;MACAC;MACAC;MACAC;MACAC;SACG,MAAML;;EAEb;EAEA,IAAIM,OAAO;AACT,WAAO,KAAKT,QAAQS,QAAQ;EAC9B;;EAGA,IAAYC,iBAA4B;AACtC,SAAKb,kBAAkB,KAAKC,YAAYR,WAAW,KAAKmB,IAAI,EAAEP,UAAU,GAAG,KAAKA,SAAS,UAAU;AACnG,WAAO,KAAKL;EACd;;EAGA,IAAYc,UAAqB;AAC/B,SAAKb,WAAW,KAAKA,YAAYR,WAAW,KAAKmB,IAAI,EAAEP,UAAU,KAAKA,SAAS;AAC/E,WAAO,KAAKJ;EACd;EAEmBc,aAA+C;AAChE,UAAMC,QAAQ,oBAAIC,IAAAA;AAClB,SAAKC,QAAQC,IAAI,wBAAwB,KAAKL,QAAQM,MAAM,EAAE;AAC9D,WAAOC,OAAOC,QAAQ,KAAKR,QAAQS,OAAM,CAAA,EACtCC,IAAI,CAAC,CAAA,EAAGC,KAAAA,MAAWA,KAAAA,EACnBC,OAAO,CAACC,YAAAA;AACP,UAAIX,MAAMY,IAAID,QAAQE,KAAK,GAAG;AAC5B,eAAO;MACT,OAAO;AACLb,cAAMc,IAAIH,QAAQE,KAAK;AACvB,eAAO;MACT;IACF,CAAA,EACCE,KAAK,CAACC,GAAGC,MAAMD,EAAEE,aAAaD,EAAEC,UAAU,EAC1CV,IAAIG,CAAAA,YAAW,KAAKQ,kBAAkBR,OAAAA,CAAAA;EAC3C;EAEmBS,eAAqC;AACtD,SAAKlB,QAAQC,IAAI,wBAAwB,KAAKL,QAAQM,MAAM,EAAE;AAC9D,SAAKN,QAAQuB,MAAK;AAClB,WAAO,KAAKC,KAAK,WAAW;MAAEC,KAAK;IAAK,CAAA;EAC1C;EAEA,MAAyBC,gBAAmD;AAC1E,SAAKtB,QAAQC,IAAI,wBAAwB,KAAKL,QAAQM,MAAM,EAAE;AAC9D,UAAMqB,WAAW,MAAM,KAAKC,IAAG;AAC/BC,aAASF,SAASrB,SAAS,GAAG,MAAM,mBAAA;AACpC,UAAMwB,WAAW,MAAMC,QAAQC,WAC7BzB,OAAO0B,QAAQ,MAAM,KAAKC,iBAAgB,GAAIC,UAAU,CAAA,CAAE,GAAGzB,IAAI,OAAO0B,WAAAA;AACtE,YAAMC,eAAqC;QAAEC,QAAQ1C;MAA2B;AAChF,YAAM2C,QAAQ,MAAM,KAAKC,UAAUH,cAAcV,QAAAA;AACjD,cAAQ,MAAMS,QAAQG,MAAMA,MAAM,CAAA,GAAIA,MAAM,CAAA,CAAE,KAAK,CAAA;IACrD,CAAA,CAAA,GACC3B,OAAO6B,MAAAA;AAEV,UAAM,KAAKlB,MAAK;AAChB,WAAQO,QAAQlB,OAAO8B,SAAAA,EAAWhC,IAAIiC,CAAAA,WAAUA,OAAOhC,KAAK,EAAGC,OAAO6B,MAAAA;EACxE;EAEA,MAAyBG,cAAcC,QAAiC;AACtE,YACE,MAAMd,QAAQH,IACZiB,OAAOnC,IAAI,CAACoC,SAAAA;AACV,WAAK9C,QAAQ+C,OAAOD,IAAAA;AACpB,aAAOA;IACT,CAAA,CAAA,GAEFlC,OAAO6B,MAAAA;EACX;EAEUO,cACRC,QAAwB,OACxBC,QAAgB,IAChBC,QACmB;AACnB,UAAMC,aAAaD,SAAU,KAAKnD,QAAQqD,IAAIF,MAAAA,GAAyCpC,QAAQuC;AAC/F,UAAMpD,QAAQ,oBAAIC,IAAAA;AAClB,UAAMwB,WAA8BpB,OAAOC,QAAQ,KAAKR,QAAQS,OAAM,CAAA,EACnEC,IAAI,CAAC,CAAA,EAAGC,KAAAA,MAAWA,KAAAA,EACnBC,OAAO,CAACC,YAAAA;AACP,UAAIX,MAAMY,IAAID,QAAQE,KAAK,GAAG;AAC5B,eAAO;MACT,OAAO;AACLb,cAAMc,IAAIH,QAAQE,KAAK;AACvB,eAAO;MACT;IACF,CAAA,EACCE,KAAK,CAACC,GAAGC,MAAAA;AACR,aAAO8B,UAAU,QAAQ/B,EAAEE,aAAaD,EAAEC,aAAaD,EAAEC,aAAaF,EAAEE;IAC1E,CAAA;AACF,QAAIgC,YAAY;AACd,YAAMG,QAAQ5B,SAAS6B,UAAU3C,CAAAA,YAAWA,QAAQE,UAAUqC,UAAAA;AAC9D,UAAIG,UAAU,IAAI;AAChB,eAAO5B,SAAS8B,MAAMF,QAAQ,GAAGA,QAAQ,IAAIL,KAAAA;MAC/C;IACF;AACA,WAAOvB,SAAS8B,MAAM,GAAGP,KAAAA;EAC3B;EAEmBQ,WAAWb,QAAiD;AAC7E,UAAM3C,QAAQ,oBAAIC,IAAAA;AAClB,WACE0C,OAAOnC,IAAI,CAACoC,SAAAA;AACV,aAAO,KAAK9C,QAAQqD,IAAIP,IAAAA;IAC1B,CAAA,EACAlC,OAAO6B,MAAAA,EACN7B,OAAO,CAACC,YAAAA;AACP,UAAIX,MAAMY,IAAID,QAAQE,KAAK,GAAG;AAC5B,eAAO;MACT,OAAO;AACLb,cAAMc,IAAIH,QAAQE,KAAK;AACvB,eAAO;MACT;IACF,CAAA,EAAGL,IAAIG,CAAAA,YAAW,KAAKQ,kBAAkBR,OAAAA,CAAAA;EAC7C;EAEA,MAAyB8C,cAAchC,UAAiD;AACtF,QAAIiC,YAAYC,KAAKC,IAAG;AACxB,UAAMC,QAAQ,MAAMC,eAAeC,UAAUtC,QAAAA;AAC7C,WAAOoC,MAAMrD,IAAI,CAAC,CAACG,SAASiC,IAAAA,MAAK;AAC/B,YAAMoB,iBAAiB,KAAKC,eAAetD,SAAS+C,WAAAA;AACpD,YAAMjD,QAAQyD,KAAKC,UAAUH,cAAAA;AAC7BI,cAAQjE,IAAI,2BAA2B6D,cAAAA;AACvCrC,eAASlB,MAAML,SAAS,KAAKhB,cAAc,MAAM,sBAAsBwD,IAAAA,KAASnC,MAAML,MAAM,GAAG;AAC/F,WAAKN,QAAQuE,IAAIzB,MAAMoB,cAAAA;AACvB,WAAKlE,QAAQuE,IAAI1D,QAAQE,OAAOmD,cAAAA;AAChC,aAAOrD;IACT,CAAA;EACF;EAEmB2D,YAAYC,SAA+D;AAC5F,UAAM,EACJvB,OAAOC,QAAQF,MAAK,IAClBwB,WAAW,CAAC;AAChB,WAAO,KAAKzB,cAAcC,OAAOC,SAAS,IAAIC,MAAAA;EAChD;EAEA,MAAyBuB,eAAe;AACtC,UAAM,MAAMA,aAAAA;AACZ,WAAO;EACT;EAEQP,eAA4CtD,SAAsBO,YAAwC;AAChH,WAAO;MAAE,GAAGP;MAASO;IAAW;EAClC;EAEQC,kBAA+CR,SAA0C;AAE/F,UAAM,EAAEO,YAAY,GAAGuD,KAAAA,IAAS9D;AAChC,WAAO8D;EACT;AACF;","names":["assertEx","exists","fulfilled","AbstractArchivist","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistCommitQuerySchema","ArchivistDeleteQuerySchema","ArchivistInsertQuerySchema","PayloadBuilder","store","storeTypes","store","StorageArchivistConfigSchema","StorageArchivist","AbstractArchivist","configSchemas","defaultConfigSchema","_privateStorage","_storage","maxEntries","config","maxEntrySize","namespace","queries","ArchivistAllQuerySchema","ArchivistDeleteQuerySchema","ArchivistClearQuerySchema","ArchivistInsertQuerySchema","ArchivistCommitQuerySchema","type","privateStorage","storage","allHandler","found","Set","logger","log","length","Object","entries","getAll","map","value","filter","payload","has","$hash","add","sort","a","b","_timestamp","removeStorageMeta","clearHandler","clear","emit","mod","commitHandler","payloads","all","assertEx","settled","Promise","allSettled","values","parentArchivists","commit","parent","queryPayload","schema","query","bindQuery","exists","fulfilled","result","deleteHandler","hashes","hash","remove","getFromOffset","order","limit","offset","offsetHash","get","undefined","index","findIndex","slice","getHandler","insertHandler","timestamp","Date","now","pairs","PayloadBuilder","hashPairs","storagePayload","addStorageMeta","JSON","stringify","console","set","nextHandler","options","startHandler","rest"]}
+{"version":3,"sources":["../../src/StorageArchivist.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash, Hex } from '@xylabs/hex'\nimport type { Promisable, PromisableArray } from '@xylabs/promise'\nimport { fulfilled } from '@xylabs/promise'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport type {\n  ArchivistConfig,\n  ArchivistInsertQuery,\n  ArchivistInstance,\n  ArchivistModuleEventData,\n  ArchivistNextOptions,\n  ArchivistParams,\n} from '@xyo-network/archivist-model'\nimport {\n  ArchivistAllQuerySchema,\n  ArchivistClearQuerySchema,\n  ArchivistCommitQuerySchema,\n  ArchivistDeleteQuerySchema,\n  ArchivistInsertQuerySchema,\n} from '@xyo-network/archivist-model'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport type { AnyConfigSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema, WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport type { StoreBase, StoreType } from 'store2'\nimport store from 'store2'\n\nconst storeTypes = store as unknown as StoreType\n\nexport type StorageArchivistConfigSchema = 'network.xyo.archivist.storage.config'\nexport const StorageArchivistConfigSchema: StorageArchivistConfigSchema = 'network.xyo.archivist.storage.config'\n\nexport type StorageArchivistConfig = ArchivistConfig<{\n  maxEntries?: number\n  maxEntrySize?: number\n  namespace?: string\n  schema: StorageArchivistConfigSchema\n  type?: 'local' | 'session' | 'page'\n}>\n\nexport type StorageArchivistParams = ArchivistParams<AnyConfigSchema<StorageArchivistConfig>>\nexport class StorageArchivist<\n  TParams extends StorageArchivistParams = StorageArchivistParams,\n  TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n>\n  extends AbstractArchivist<TParams, TEventData>\n  implements ArchivistInstance {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, StorageArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = StorageArchivistConfigSchema\n\n  private _privateStorage: StoreBase | undefined\n  private _storage: StoreBase | undefined\n\n  get maxEntries() {\n    return this.config?.maxEntries ?? 1000\n  }\n\n  get maxEntrySize() {\n    return this.config?.maxEntrySize ?? 16_000\n  }\n\n  get namespace() {\n    return this.config?.namespace ?? 'xyo-archivist'\n  }\n\n  override get queries(): string[] {\n    return [\n      ArchivistAllQuerySchema,\n      ArchivistDeleteQuerySchema,\n      ArchivistClearQuerySchema,\n      ArchivistInsertQuerySchema,\n      ArchivistCommitQuerySchema,\n      ...super.queries,\n    ]\n  }\n\n  get type() {\n    return this.config?.type ?? 'local'\n  }\n\n  /* This has to be a getter so that it can access it during construction */\n  private get privateStorage(): StoreBase {\n    this._privateStorage = this._storage ?? storeTypes[this.type].namespace(`${this.namespace}|private`)\n    return this._privateStorage\n  }\n\n  /* This has to be a getter so that it can access it during construction */\n  private get storage(): StoreBase {\n    this._storage = this._storage ?? storeTypes[this.type].namespace(this.namespace)\n    return this._storage\n  }\n\n  protected override allHandler(): PromisableArray<WithStorageMeta<Payload>> {\n    const found = new Set<string>()\n    this.logger?.log(`this.storage.length: ${this.storage.length}`)\n    return Object.entries(this.storage.getAll())\n      .map(([, value]) => value)\n      .filter((payload) => {\n        if (found.has(payload._dataHash)) {\n          return false\n        } else {\n          found.add(payload._dataHash)\n          return true\n        }\n      })\n      .sort(PayloadBuilder.compareStorageMeta)\n      .map(payload => PayloadBuilder.omitStorageMeta(payload))\n  }\n\n  protected override clearHandler(): void | Promise<void> {\n    this.logger?.log(`this.storage.length: ${this.storage.length}`)\n    this.storage.clear()\n    return this.emit('cleared', { mod: this })\n  }\n\n  protected override async commitHandler(): Promise<BoundWitness[]> {\n    this.logger?.log(`this.storage.length: ${this.storage.length}`)\n    const payloads = await this.all()\n    assertEx(payloads.length > 0, () => 'Nothing to commit')\n    const settled = (await Promise.allSettled(\n      Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n        const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n        const query = await this.bindQuery(queryPayload, payloads)\n        return (await parent?.query(query[0], query[1]))?.[0]\n      }),\n    )).filter(exists)\n    // TODO - rather than clear, delete the payloads that come back as successfully inserted\n    await this.clear()\n    return (settled.filter(fulfilled).map(result => result.value)).filter(exists)\n  }\n\n  protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {\n    return (\n      await Promise.all(\n        hashes.map((hash) => {\n          this.storage.remove(hash)\n          return hash\n        }),\n      )\n    ).filter(exists)\n  }\n\n  protected getFromCursor(\n    order: 'asc' | 'desc' = 'asc',\n    limit: number = 10,\n    cursor?: Hex,\n  ): WithStorageMeta[] {\n    const all = Object.values(this.storage.getAll()) as WithStorageMeta[]\n    const payloads: WithStorageMeta[] = all\n      .map(value => value)\n      .sort((a, b) => {\n        return order === 'asc' ? PayloadBuilder.compareStorageMeta(a, b) : PayloadBuilder.compareStorageMeta(b, a)\n      })\n    const index = payloads.findIndex(payload => payload._sequence === cursor)\n    if (index !== -1) {\n      return payloads.slice(index + 1, index + 1 + limit)\n    }\n    return payloads.slice(0, limit)\n  }\n\n  protected override getHandler(hashes: string[]): Promisable<WithStorageMeta<Payload>[]> {\n    const found = new Set<string>()\n    return (\n      hashes.map((hash) => {\n        return this.storage.get(hash)\n      })\n    ).filter(exists)\n      .filter((payload) => {\n        if (found.has(payload._dataHash)) {\n          return false\n        } else {\n          found.add(payload._dataHash)\n          return true\n        }\n      }).map(payload => PayloadBuilder.omitStorageMeta(payload))\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<WithStorageMeta<Payload>[]> {\n    const storagePayloads = (await PayloadBuilder.addStorageMeta(payloads)).sort(PayloadBuilder.compareStorageMeta)\n    return await Promise.all(storagePayloads.map((storagePayload) => {\n      const value = JSON.stringify(storagePayload)\n      // console.log('insert.storagePayloads:', storagePayload)\n      assertEx(value.length < this.maxEntrySize, () => `Payload too large [${storagePayload._hash}, ${value.length}]`)\n      this.storage.set(storagePayload._hash, storagePayload)\n      this.storage.set(storagePayload._dataHash, storagePayload)\n      return storagePayload\n    }))\n  }\n\n  protected override nextHandler(options?: ArchivistNextOptions): Promisable<WithStorageMeta<Payload>[]> {\n    const {\n      limit, cursor, order,\n    } = options ?? {}\n    return this.getFromCursor(order, limit ?? 10, cursor)\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    return true\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,iBAAiB;AAC1B,SAASC,yBAAyB;AASlC,SACEC,yBACAC,2BACAC,4BACAC,4BACAC,kCACK;AAGP,SAASC,sBAAsB;AAK/B,OAAOC,WAAW;AAElB,IAAMC,aAAaC;AAGZ,IAAMC,+BAA6D;AAWnE,IAAMC,mBAAN,cAIGC,kBAAAA;EAhDV,OAgDUA;;;EAER,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE/CK;EACAC;EAER,IAAIC,aAAa;AACf,WAAO,KAAKC,QAAQD,cAAc;EACpC;EAEA,IAAIE,eAAe;AACjB,WAAO,KAAKD,QAAQC,gBAAgB;EACtC;EAEA,IAAIC,YAAY;AACd,WAAO,KAAKF,QAAQE,aAAa;EACnC;EAEA,IAAaC,UAAoB;AAC/B,WAAO;MACLC;MACAC;MACAC;MACAC;MACAC;SACG,MAAML;;EAEb;EAEA,IAAIM,OAAO;AACT,WAAO,KAAKT,QAAQS,QAAQ;EAC9B;;EAGA,IAAYC,iBAA4B;AACtC,SAAKb,kBAAkB,KAAKC,YAAYR,WAAW,KAAKmB,IAAI,EAAEP,UAAU,GAAG,KAAKA,SAAS,UAAU;AACnG,WAAO,KAAKL;EACd;;EAGA,IAAYc,UAAqB;AAC/B,SAAKb,WAAW,KAAKA,YAAYR,WAAW,KAAKmB,IAAI,EAAEP,UAAU,KAAKA,SAAS;AAC/E,WAAO,KAAKJ;EACd;EAEmBc,aAAwD;AACzE,UAAMC,QAAQ,oBAAIC,IAAAA;AAClB,SAAKC,QAAQC,IAAI,wBAAwB,KAAKL,QAAQM,MAAM,EAAE;AAC9D,WAAOC,OAAOC,QAAQ,KAAKR,QAAQS,OAAM,CAAA,EACtCC,IAAI,CAAC,CAAA,EAAGC,KAAAA,MAAWA,KAAAA,EACnBC,OAAO,CAACC,YAAAA;AACP,UAAIX,MAAMY,IAAID,QAAQE,SAAS,GAAG;AAChC,eAAO;MACT,OAAO;AACLb,cAAMc,IAAIH,QAAQE,SAAS;AAC3B,eAAO;MACT;IACF,CAAA,EACCE,KAAKC,eAAeC,kBAAkB,EACtCT,IAAIG,CAAAA,YAAWK,eAAeE,gBAAgBP,OAAAA,CAAAA;EACnD;EAEmBQ,eAAqC;AACtD,SAAKjB,QAAQC,IAAI,wBAAwB,KAAKL,QAAQM,MAAM,EAAE;AAC9D,SAAKN,QAAQsB,MAAK;AAClB,WAAO,KAAKC,KAAK,WAAW;MAAEC,KAAK;IAAK,CAAA;EAC1C;EAEA,MAAyBC,gBAAyC;AAChE,SAAKrB,QAAQC,IAAI,wBAAwB,KAAKL,QAAQM,MAAM,EAAE;AAC9D,UAAMoB,WAAW,MAAM,KAAKC,IAAG;AAC/BC,aAASF,SAASpB,SAAS,GAAG,MAAM,mBAAA;AACpC,UAAMuB,WAAW,MAAMC,QAAQC,WAC7BxB,OAAOyB,QAAQ,MAAM,KAAKC,iBAAgB,GAAIC,UAAU,CAAA,CAAE,GAAGxB,IAAI,OAAOyB,WAAAA;AACtE,YAAMC,eAAqC;QAAEC,QAAQzC;MAA2B;AAChF,YAAM0C,QAAQ,MAAM,KAAKC,UAAUH,cAAcV,QAAAA;AACjD,cAAQ,MAAMS,QAAQG,MAAMA,MAAM,CAAA,GAAIA,MAAM,CAAA,CAAE,KAAK,CAAA;IACrD,CAAA,CAAA,GACC1B,OAAO4B,MAAAA;AAEV,UAAM,KAAKlB,MAAK;AAChB,WAAQO,QAAQjB,OAAO6B,SAAAA,EAAW/B,IAAIgC,CAAAA,WAAUA,OAAO/B,KAAK,EAAGC,OAAO4B,MAAAA;EACxE;EAEA,MAAyBG,cAAcC,QAAiC;AACtE,YACE,MAAMd,QAAQH,IACZiB,OAAOlC,IAAI,CAACmC,SAAAA;AACV,WAAK7C,QAAQ8C,OAAOD,IAAAA;AACpB,aAAOA;IACT,CAAA,CAAA,GAEFjC,OAAO4B,MAAAA;EACX;EAEUO,cACRC,QAAwB,OACxBC,QAAgB,IAChBC,QACmB;AACnB,UAAMvB,MAAMpB,OAAOyB,OAAO,KAAKhC,QAAQS,OAAM,CAAA;AAC7C,UAAMiB,WAA8BC,IACjCjB,IAAIC,CAAAA,UAASA,KAAAA,EACbM,KAAK,CAACkC,GAAGC,MAAAA;AACR,aAAOJ,UAAU,QAAQ9B,eAAeC,mBAAmBgC,GAAGC,CAAAA,IAAKlC,eAAeC,mBAAmBiC,GAAGD,CAAAA;IAC1G,CAAA;AACF,UAAME,QAAQ3B,SAAS4B,UAAUzC,CAAAA,YAAWA,QAAQ0C,cAAcL,MAAAA;AAClE,QAAIG,UAAU,IAAI;AAChB,aAAO3B,SAAS8B,MAAMH,QAAQ,GAAGA,QAAQ,IAAIJ,KAAAA;IAC/C;AACA,WAAOvB,SAAS8B,MAAM,GAAGP,KAAAA;EAC3B;EAEmBQ,WAAWb,QAA0D;AACtF,UAAM1C,QAAQ,oBAAIC,IAAAA;AAClB,WACEyC,OAAOlC,IAAI,CAACmC,SAAAA;AACV,aAAO,KAAK7C,QAAQ0D,IAAIb,IAAAA;IAC1B,CAAA,EACAjC,OAAO4B,MAAAA,EACN5B,OAAO,CAACC,YAAAA;AACP,UAAIX,MAAMY,IAAID,QAAQE,SAAS,GAAG;AAChC,eAAO;MACT,OAAO;AACLb,cAAMc,IAAIH,QAAQE,SAAS;AAC3B,eAAO;MACT;IACF,CAAA,EAAGL,IAAIG,CAAAA,YAAWK,eAAeE,gBAAgBP,OAAAA,CAAAA;EACrD;EAEA,MAAyB8C,cAAcjC,UAA0D;AAC/F,UAAMkC,mBAAmB,MAAM1C,eAAe2C,eAAenC,QAAAA,GAAWT,KAAKC,eAAeC,kBAAkB;AAC9G,WAAO,MAAMW,QAAQH,IAAIiC,gBAAgBlD,IAAI,CAACoD,mBAAAA;AAC5C,YAAMnD,QAAQoD,KAAKC,UAAUF,cAAAA;AAE7BlC,eAASjB,MAAML,SAAS,KAAKhB,cAAc,MAAM,sBAAsBwE,eAAeG,KAAK,KAAKtD,MAAML,MAAM,GAAG;AAC/G,WAAKN,QAAQkE,IAAIJ,eAAeG,OAAOH,cAAAA;AACvC,WAAK9D,QAAQkE,IAAIJ,eAAe/C,WAAW+C,cAAAA;AAC3C,aAAOA;IACT,CAAA,CAAA;EACF;EAEmBK,YAAYC,SAAwE;AACrG,UAAM,EACJnB,OAAOC,QAAQF,MAAK,IAClBoB,WAAW,CAAC;AAChB,WAAO,KAAKrB,cAAcC,OAAOC,SAAS,IAAIC,MAAAA;EAChD;EAEA,MAAyBmB,eAAe;AACtC,UAAM,MAAMA,aAAAA;AACZ,WAAO;EACT;AACF;","names":["assertEx","exists","fulfilled","AbstractArchivist","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistCommitQuerySchema","ArchivistDeleteQuerySchema","ArchivistInsertQuerySchema","PayloadBuilder","store","storeTypes","store","StorageArchivistConfigSchema","StorageArchivist","AbstractArchivist","configSchemas","defaultConfigSchema","_privateStorage","_storage","maxEntries","config","maxEntrySize","namespace","queries","ArchivistAllQuerySchema","ArchivistDeleteQuerySchema","ArchivistClearQuerySchema","ArchivistInsertQuerySchema","ArchivistCommitQuerySchema","type","privateStorage","storage","allHandler","found","Set","logger","log","length","Object","entries","getAll","map","value","filter","payload","has","_dataHash","add","sort","PayloadBuilder","compareStorageMeta","omitStorageMeta","clearHandler","clear","emit","mod","commitHandler","payloads","all","assertEx","settled","Promise","allSettled","values","parentArchivists","commit","parent","queryPayload","schema","query","bindQuery","exists","fulfilled","result","deleteHandler","hashes","hash","remove","getFromCursor","order","limit","cursor","a","b","index","findIndex","_sequence","slice","getHandler","get","insertHandler","storagePayloads","addStorageMeta","storagePayload","JSON","stringify","_hash","set","nextHandler","options","startHandler"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/archivist-storage",
-  "version": "3.5.2",
+  "version": "3.6.0-rc.10",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -29,33 +29,34 @@
   "module": "dist/neutral/index.mjs",
   "types": "dist/neutral/index.d.ts",
   "dependencies": {
-    "@xylabs/assert": "^4.4.9",
-    "@xylabs/exists": "^4.4.9",
-    "@xylabs/hex": "^4.4.9",
-    "@xylabs/promise": "^4.4.9",
-    "@xyo-network/archivist-abstract": "^3.5.2",
-    "@xyo-network/archivist-model": "^3.5.2",
-    "@xyo-network/boundwitness-model": "^3.5.2",
-    "@xyo-network/module-model": "^3.5.2",
-    "@xyo-network/payload-builder": "^3.5.2",
-    "@xyo-network/payload-model": "^3.5.2",
+    "@xylabs/assert": "^4.4.21",
+    "@xylabs/exists": "^4.4.21",
+    "@xylabs/hex": "^4.4.21",
+    "@xylabs/promise": "^4.4.21",
+    "@xyo-network/archivist-abstract": "^3.6.0-rc.10",
+    "@xyo-network/archivist-model": "^3.6.0-rc.10",
+    "@xyo-network/boundwitness-model": "^3.6.0-rc.10",
+    "@xyo-network/module-model": "^3.6.0-rc.10",
+    "@xyo-network/payload-builder": "^3.6.0-rc.10",
+    "@xyo-network/payload-model": "^3.6.0-rc.10",
     "store2": "^2.14.3"
   },
   "devDependencies": {
-    "@xylabs/delay": "^4.4.9",
-    "@xylabs/ts-scripts-yarn3": "^4.2.4",
-    "@xylabs/tsconfig": "^4.2.4",
-    "@xylabs/vitest-extended": "^4.4.9",
-    "@xyo-network/account": "^3.5.2",
-    "@xyo-network/archivist-memory": "^3.5.2",
-    "@xyo-network/boundwitness-wrapper": "^3.5.2",
-    "@xyo-network/id-payload-plugin": "^3.5.2",
-    "@xyo-network/node-memory": "^3.5.2",
-    "@xyo-network/payload-wrapper": "^3.5.2",
+    "@xylabs/delay": "^4.4.21",
+    "@xylabs/ts-scripts-yarn3": "^4.2.6",
+    "@xylabs/tsconfig": "^4.2.6",
+    "@xylabs/vitest-extended": "^4.4.21",
+    "@xyo-network/account": "^3.6.0-rc.10",
+    "@xyo-network/archivist-memory": "^3.6.0-rc.10",
+    "@xyo-network/boundwitness-wrapper": "^3.6.0-rc.10",
+    "@xyo-network/id-payload-plugin": "^3.6.0-rc.10",
+    "@xyo-network/node-memory": "^3.6.0-rc.10",
+    "@xyo-network/payload-wrapper": "^3.6.0-rc.10",
     "typescript": "^5.7.2",
-    "vitest": "^2.1.5"
+    "vitest": "^2.1.8"
   },
   "publishConfig": {
     "access": "public"
-  }
+  },
+  "stableVersion": "3.5.2"
 }

package/src/StorageArchivist.ts

@@ -1,6 +1,6 @@
 import { assertEx } from '@xylabs/assert'
 import { exists } from '@xylabs/exists'
-import type { Hash } from '@xylabs/hex'
+import type { Hash, Hex } from '@xylabs/hex'
 import type { Promisable, PromisableArray } from '@xylabs/promise'
 import { fulfilled } from '@xylabs/promise'
 import { AbstractArchivist } from '@xyo-network/archivist-abstract'
@@ -23,15 +23,13 @@ import type { BoundWitness } from '@xyo-network/boundwitness-model'
 import type { AnyConfigSchema } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
 import type {
-  Payload, PayloadWithMeta, Schema, WithMeta,
+  Payload, Schema, WithStorageMeta,
 } from '@xyo-network/payload-model'
 import type { StoreBase, StoreType } from 'store2'
 import store from 'store2'
 
 const storeTypes = store as unknown as StoreType
 
-type WithStorageMeta<T extends Payload = Payload> = WithMeta<T> & { _timestamp: number }
-
 export type StorageArchivistConfigSchema = 'network.xyo.archivist.storage.config'
 export const StorageArchivistConfigSchema: StorageArchivistConfigSchema = 'network.xyo.archivist.storage.config'
 
@@ -95,21 +93,21 @@ export class StorageArchivist<
     return this._storage
   }
 
-  protected override allHandler(): PromisableArray<PayloadWithMeta> {
+  protected override allHandler(): PromisableArray<WithStorageMeta<Payload>> {
     const found = new Set<string>()
     this.logger?.log(`this.storage.length: ${this.storage.length}`)
     return Object.entries(this.storage.getAll())
       .map(([, value]) => value)
       .filter((payload) => {
-        if (found.has(payload.$hash)) {
+        if (found.has(payload._dataHash)) {
           return false
         } else {
-          found.add(payload.$hash)
+          found.add(payload._dataHash)
           return true
         }
       })
-      .sort((a, b) => a._timestamp - b._timestamp)
-      .map(payload => this.removeStorageMeta(payload))
+      .sort(PayloadBuilder.compareStorageMeta)
+      .map(payload => PayloadBuilder.omitStorageMeta(payload))
   }
 
   protected override clearHandler(): void | Promise<void> {
@@ -118,7 +116,7 @@ export class StorageArchivist<
     return this.emit('cleared', { mod: this })
   }
 
-  protected override async commitHandler(): Promise<WithMeta<BoundWitness>[]> {
+  protected override async commitHandler(): Promise<BoundWitness[]> {
     this.logger?.log(`this.storage.length: ${this.storage.length}`)
     const payloads = await this.all()
     assertEx(payloads.length > 0, () => 'Nothing to commit')
@@ -145,36 +143,25 @@ export class StorageArchivist<
     ).filter(exists)
   }
 
-  protected getFromOffset(
+  protected getFromCursor(
     order: 'asc' | 'desc' = 'asc',
     limit: number = 10,
-    offset?: Hash,
+    cursor?: Hex,
   ): WithStorageMeta[] {
-    const offsetHash = offset ? (this.storage.get(offset) as PayloadWithMeta | undefined)?.$hash : undefined
-    const found = new Set<string>()
-    const payloads: WithStorageMeta[] = Object.entries(this.storage.getAll())
-      .map(([, value]) => value)
-      .filter((payload) => {
-        if (found.has(payload.$hash)) {
-          return false
-        } else {
-          found.add(payload.$hash)
-          return true
-        }
-      })
+    const all = Object.values(this.storage.getAll()) as WithStorageMeta[]
+    const payloads: WithStorageMeta[] = all
+      .map(value => value)
       .sort((a, b) => {
-        return order === 'asc' ? a._timestamp - b._timestamp : b._timestamp - a._timestamp
+        return order === 'asc' ? PayloadBuilder.compareStorageMeta(a, b) : PayloadBuilder.compareStorageMeta(b, a)
       })
-    if (offsetHash) {
-      const index = payloads.findIndex(payload => payload.$hash === offsetHash)
-      if (index !== -1) {
-        return payloads.slice(index + 1, index + 1 + limit)
-      }
+    const index = payloads.findIndex(payload => payload._sequence === cursor)
+    if (index !== -1) {
+      return payloads.slice(index + 1, index + 1 + limit)
     }
     return payloads.slice(0, limit)
   }
 
-  protected override getHandler(hashes: string[]): Promisable<PayloadWithMeta[]> {
+  protected override getHandler(hashes: string[]): Promisable<WithStorageMeta<Payload>[]> {
     const found = new Set<string>()
     return (
       hashes.map((hash) => {
@@ -182,48 +169,36 @@ export class StorageArchivist<
       })
     ).filter(exists)
       .filter((payload) => {
-        if (found.has(payload.$hash)) {
+        if (found.has(payload._dataHash)) {
           return false
         } else {
-          found.add(payload.$hash)
+          found.add(payload._dataHash)
           return true
         }
-      }).map(payload => this.removeStorageMeta(payload))
+      }).map(payload => PayloadBuilder.omitStorageMeta(payload))
   }
 
-  protected override async insertHandler(payloads: Payload[]): Promise<PayloadWithMeta[]> {
-    let timestamp = Date.now()
-    const pairs = await PayloadBuilder.hashPairs(payloads)
-    return pairs.map(([payload, hash]) => {
-      const storagePayload = this.addStorageMeta(payload, timestamp++)
+  protected override async insertHandler(payloads: Payload[]): Promise<WithStorageMeta<Payload>[]> {
+    const storagePayloads = (await PayloadBuilder.addStorageMeta(payloads)).sort(PayloadBuilder.compareStorageMeta)
+    return await Promise.all(storagePayloads.map((storagePayload) => {
       const value = JSON.stringify(storagePayload)
-      console.log('insert.storagePayloads:', storagePayload)
-      assertEx(value.length < this.maxEntrySize, () => `Payload too large [${hash}, ${value.length}]`)
-      this.storage.set(hash, storagePayload)
-      this.storage.set(payload.$hash, storagePayload)
-      return payload
-    })
+      // console.log('insert.storagePayloads:', storagePayload)
+      assertEx(value.length < this.maxEntrySize, () => `Payload too large [${storagePayload._hash}, ${value.length}]`)
+      this.storage.set(storagePayload._hash, storagePayload)
+      this.storage.set(storagePayload._dataHash, storagePayload)
+      return storagePayload
+    }))
   }
 
-  protected override nextHandler(options?: ArchivistNextOptions): Promisable<PayloadWithMeta[]> {
+  protected override nextHandler(options?: ArchivistNextOptions): Promisable<WithStorageMeta<Payload>[]> {
     const {
-      limit, offset, order,
+      limit, cursor, order,
     } = options ?? {}
-    return this.getFromOffset(order, limit ?? 10, offset)
+    return this.getFromCursor(order, limit ?? 10, cursor)
   }
 
   protected override async startHandler() {
     await super.startHandler()
     return true
   }
-
-  private addStorageMeta<T extends Payload = Payload>(payload: WithMeta<T>, _timestamp: number): WithStorageMeta<T> {
-    return { ...payload, _timestamp }
-  }
-
-  private removeStorageMeta<T extends Payload = Payload>(payload: WithStorageMeta<T>): WithMeta<T> {
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    const { _timestamp, ...rest } = payload
-    return rest as WithMeta<T>
-  }
 }