@xyo-network/archivist-memory 3.5.1 → 3.6.0-rc.1

package/dist/neutral/MemoryArchivist.d.ts

@@ -5,7 +5,7 @@ import { AbstractArchivist, WithStorageMeta } from '@xyo-network/archivist-abstr
 import { ArchivistConfig, ArchivistModuleEventData, ArchivistNextOptions, AttachableArchivistInstance } from '@xyo-network/archivist-model';
 import { BoundWitness } from '@xyo-network/boundwitness-model';
 import { AnyConfigSchema, ModuleInstance, ModuleParams } from '@xyo-network/module-model';
-import { Payload, PayloadWithMeta, Schema, WithMeta } from '@xyo-network/payload-model';
+import { Payload, Schema } from '@xyo-network/payload-model';
 import { LRUCache } from 'lru-cache';
 export type MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config';
 export declare const MemoryArchivistConfigSchema: MemoryArchivistConfigSchema;
@@ -19,16 +19,18 @@ export declare class MemoryArchivist<TParams extends MemoryArchivistParams<AnyCo
     private _cache?;
     private _dataHashIndex?;
     get queries(): string[];
-    protected get cache(): LRUCache<Lowercase<string>, WithStorageMeta<PayloadWithMeta>, unknown>;
+    protected get cache(): LRUCache<Lowercase<string>, WithStorageMeta<{
+        schema: string;
+    } & import("@xyo-network/payload-model").PayloadFields>, unknown>;
     protected get dataHashIndex(): LRUCache<Lowercase<string>, Lowercase<string>, unknown>;
     protected get max(): number;
-    protected allHandler(): Promisable<PayloadWithMeta[]>;
+    protected allHandler(): Promisable<Payload[]>;
     protected clearHandler(): void | Promise<void>;
-    protected commitHandler(): Promise<WithMeta<BoundWitness>[]>;
-    protected deleteHandler(hashes: Hash[]): Promisable<Hash[]>;
-    protected getHandler(hashes: Hash[]): Promisable<PayloadWithMeta[]>;
-    protected insertHandler(payloads: Payload[]): Promise<PayloadWithMeta[]>;
-    protected nextHandler(options?: ArchivistNextOptions): Promise<PayloadWithMeta[]>;
+    protected commitHandler(): Promise<BoundWitness[]>;
+    protected deleteHandler(hashes: Hash[]): Promise<Hash[]>;
+    protected getHandler(hashes: Hash[]): Promisable<Payload[]>;
+    protected insertHandler(payloads: Payload[]): Promise<Payload[]>;
+    protected nextHandler(options?: ArchivistNextOptions): Promise<Payload[]>;
     private insertPayloadIntoCache;
     private rebuildDataHashIndex;
 }

package/dist/neutral/MemoryArchivist.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"MemoryArchivist.d.ts","sourceRoot":"","sources":["../../src/MemoryArchivist.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC5D,OAAO,EAAa,UAAU,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EACL,iBAAiB,EAAwD,eAAe,EACzF,MAAM,iCAAiC,CAAA;AACxC,OAAO,EAIL,eAAe,EAIf,wBAAwB,EACxB,oBAAoB,EAEpB,2BAA2B,EAC5B,MAAM,8BAA8B,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAC9D,OAAO,EACL,eAAe,EAAmB,cAAc,EAAE,YAAY,EAC/D,MAAM,2BAA2B,CAAA;AAElC,OAAO,EACL,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,EAC3C,MAAM,4BAA4B,CAAA;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAEpC,MAAM,MAAM,2BAA2B,GAAG,qCAAqC,CAAA;AAC/E,eAAO,MAAM,2BAA2B,EAAE,2BAAmE,CAAA;AAE7G,MAAM,MAAM,qBAAqB,CAAC,OAAO,SAAS,OAAO,GAAG,WAAW,GAAG,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,MAAM,GAAG,IAAI,GAAG,IAAI,IAAI,eAAe,CAC5I,cAAc,CACZ;IACE,GAAG,CAAC,EAAE,MAAM,CAAA;CACb,EACD,OAAO,CACR,EACD,OAAO,SAAS,MAAM,GAAG,OAAO,GAAG,2BAA2B,GAAG,eAAe,CAAC,QAAQ,CAAC,CAC3F,CAAA;AAED,MAAM,MAAM,qBAAqB,CAAC,OAAO,SAAS,eAAe,CAAC,qBAAqB,CAAC,GAAG,eAAe,CAAC,qBAAqB,CAAC,IAC/H,YAAY,CAAC,OAAO,CAAC,CAAA;AACvB,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,CAAC,eAAe,CAAC,qBAAqB,CAAC,CAAC,GAAG,qBAAqB,EACrG,UAAU,SAAS,wBAAwB,GAAG,wBAAwB,CAEtE,SAAQ,iBAAiB,CAAC,OAAO,EAAE,UAAU,CAC7C,YAAW,2BAA2B,EAAE,cAAc;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAwD;IACxG,gBAAyB,mBAAmB,EAAE,MAAM,CAA8B;IAElF,OAAO,CAAC,MAAM,CAAC,CAAkD;IACjE,OAAO,CAAC,cAAc,CAAC,CAAsB;IAE7C,IAAa,OAAO,aAUnB;IAED,SAAS,KAAK,KAAK,2EAGlB;IAED,SAAS,KAAK,aAAa,4DAG1B;IAED,SAAS,KAAK,GAAG,WAEhB;cAEkB,UAAU,IAAI,UAAU,CAAC,eAAe,EAAE,CAAC;cAK3C,YAAY,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;cAM9B,aAAa,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;cAaxD,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;cAcjD,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC,eAAe,EAAE,CAAC;cAWnD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;cAS9D,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAahG,OAAO,CAAC,sBAAsB;IAO9B,OAAO,CAAC,oBAAoB;CAO7B"}
+{"version":3,"file":"MemoryArchivist.d.ts","sourceRoot":"","sources":["../../src/MemoryArchivist.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC5D,OAAO,EAAa,UAAU,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAA;AACpF,OAAO,EAIL,eAAe,EAIf,wBAAwB,EACxB,oBAAoB,EAEpB,2BAA2B,EAC5B,MAAM,8BAA8B,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAC9D,OAAO,EACL,eAAe,EAAmB,cAAc,EAAE,YAAY,EAC/D,MAAM,2BAA2B,CAAA;AAElC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAEpC,MAAM,MAAM,2BAA2B,GAAG,qCAAqC,CAAA;AAC/E,eAAO,MAAM,2BAA2B,EAAE,2BAAmE,CAAA;AAE7G,MAAM,MAAM,qBAAqB,CAAC,OAAO,SAAS,OAAO,GAAG,WAAW,GAAG,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,MAAM,GAAG,IAAI,GAAG,IAAI,IAAI,eAAe,CAC5I,cAAc,CACZ;IACE,GAAG,CAAC,EAAE,MAAM,CAAA;CACb,EACD,OAAO,CACR,EACD,OAAO,SAAS,MAAM,GAAG,OAAO,GAAG,2BAA2B,GAAG,eAAe,CAAC,QAAQ,CAAC,CAC3F,CAAA;AAED,MAAM,MAAM,qBAAqB,CAAC,OAAO,SAAS,eAAe,CAAC,qBAAqB,CAAC,GAAG,eAAe,CAAC,qBAAqB,CAAC,IAC/H,YAAY,CAAC,OAAO,CAAC,CAAA;AACvB,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,CAAC,eAAe,CAAC,qBAAqB,CAAC,CAAC,GAAG,qBAAqB,EACrG,UAAU,SAAS,wBAAwB,GAAG,wBAAwB,CAEtE,SAAQ,iBAAiB,CAAC,OAAO,EAAE,UAAU,CAC7C,YAAW,2BAA2B,EAAE,cAAc;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAwD;IACxG,gBAAyB,mBAAmB,EAAE,MAAM,CAA8B;IAElF,OAAO,CAAC,MAAM,CAAC,CAA0C;IACzD,OAAO,CAAC,cAAc,CAAC,CAAsB;IAE7C,IAAa,OAAO,aAUnB;IAED,SAAS,KAAK,KAAK;;sEAGlB;IAED,SAAS,KAAK,aAAa,4DAG1B;IAED,SAAS,KAAK,GAAG,WAEhB;cAEkB,UAAU,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;cAKnC,YAAY,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;cAM9B,aAAa,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;cAaxC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;cAepD,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC;cAW3C,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;cAStD,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YAa1E,sBAAsB;YAQtB,oBAAoB;CAQnC"}

package/dist/neutral/index.mjs

@@ -5,7 +5,7 @@ var __name = (target, value) => __defProp(target, "name", { value, configurable:
 import { assertEx } from "@xylabs/assert";
 import { exists } from "@xylabs/exists";
 import { fulfilled } from "@xylabs/promise";
-import { AbstractArchivist, addStorageMeta, removeStorageMeta, sortByStorageMeta } from "@xyo-network/archivist-abstract";
+import { AbstractArchivist } from "@xyo-network/archivist-abstract";
 import { ArchivistAllQuerySchema, ArchivistClearQuerySchema, ArchivistCommitQuerySchema, ArchivistDeleteQuerySchema, ArchivistInsertQuerySchema, ArchivistNextQuerySchema } from "@xyo-network/archivist-model";
 import { creatableModule } from "@xyo-network/module-model";
 import { PayloadBuilder } from "@xyo-network/payload-builder";
@@ -18,7 +18,7 @@ function _ts_decorate(decorators, target, key, desc) {
 }
 __name(_ts_decorate, "_ts_decorate");
 var MemoryArchivistConfigSchema = "network.xyo.archivist.memory.config";
-var MemoryArchivist = class extends AbstractArchivist {
+var MemoryArchivist = class _MemoryArchivist extends AbstractArchivist {
   static {
     __name(this, "MemoryArchivist");
   }
@@ -57,7 +57,7 @@ var MemoryArchivist = class extends AbstractArchivist {
   }
   allHandler() {
     const all = this.cache.dump().map(([, item]) => item.value).filter(exists);
-    return sortByStorageMeta(all).map((payload) => removeStorageMeta(payload));
+    return _MemoryArchivist.sortByStorageMeta(all).map((payload) => _MemoryArchivist.removeStorageMeta(payload));
   }
   clearHandler() {
     this.cache.clear();
@@ -78,14 +78,15 @@ var MemoryArchivist = class extends AbstractArchivist {
     await this.clearHandler();
     return settled.filter(fulfilled).map((result) => result.value).filter(exists);
   }
-  deleteHandler(hashes) {
-    const deletedHashes = this.cache.dump().map(([key, item]) => {
-      if (hashes.includes(key) || hashes.includes(item.value.$hash)) {
+  async deleteHandler(hashes) {
+    const deletedHashes = (await Promise.all(this.cache.dump().map(async ([key, item]) => {
+      const itemValueDataHash = await PayloadBuilder.dataHash(item.value);
+      if (hashes.includes(key) || hashes.includes(itemValueDataHash)) {
         this.cache.delete(key);
         return key;
       }
-    }).filter(exists);
-    this.rebuildDataHashIndex();
+    }))).filter(exists);
+    await this.rebuildDataHashIndex();
     return deletedHashes;
   }
   getHandler(hashes) {
@@ -95,17 +96,15 @@ var MemoryArchivist = class extends AbstractArchivist {
       if (resolvedHash !== hash && !result) {
         throw new Error("Missing referenced payload");
       }
-      return removeStorageMeta(result);
+      return _MemoryArchivist.removeStorageMeta(result);
     }).filter(exists);
   }
   async insertHandler(payloads) {
-    const pairs = await PayloadBuilder.hashPairs(payloads, {
-      stamp: false
-    });
-    const insertedPayloads = pairs.map(([payload, hash], index) => {
-      return this.cache.get(hash) ?? this.insertPayloadIntoCache(payload, hash, index);
-    });
-    return removeStorageMeta(insertedPayloads);
+    const pairs = await PayloadBuilder.hashPairs(payloads);
+    const insertedPayloads = await Promise.all(pairs.map(async ([payload, hash], index) => {
+      return this.cache.get(hash) ?? await this.insertPayloadIntoCache(payload, hash, index);
+    }));
+    return _MemoryArchivist.removeStorageMeta(insertedPayloads);
   }
   async nextHandler(options) {
     const { limit, offset, order } = options ?? {};
@@ -117,19 +116,21 @@ var MemoryArchivist = class extends AbstractArchivist {
     const startIndex = offset ? allPairs.findIndex(([, hash]) => hash === offset) + 1 : 0;
     return allPairs.slice(startIndex, limit ? startIndex + limit : void 0).map(([payload]) => payload);
   }
-  insertPayloadIntoCache(payload, hash, index = 0) {
-    const withMeta = addStorageMeta(payload, index);
+  async insertPayloadIntoCache(payload, hash, index = 0) {
+    const dataHash = await PayloadBuilder.dataHash(payload);
+    const withMeta = await _MemoryArchivist.addSequencedStorageMeta(payload, index);
     this.cache.set(hash, withMeta);
-    this.dataHashIndex.set(withMeta.$hash, hash);
+    this.dataHashIndex.set(dataHash, hash);
     return withMeta;
   }
-  rebuildDataHashIndex() {
+  async rebuildDataHashIndex() {
     this._dataHashIndex = new LRUCache({
       max: this.max
     });
     const pairs = this.cache.dump();
     for (const [hash, payload] of pairs) {
-      this.dataHashIndex.set(payload.value.$hash, hash);
+      const dataHash = await PayloadBuilder.dataHash(payload.value);
+      this.dataHashIndex.set(dataHash, hash);
     }
   }
 };

package/dist/neutral/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/MemoryArchivist.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, WithAdditional } from '@xylabs/object'\nimport { fulfilled, Promisable } from '@xylabs/promise'\nimport {\n  AbstractArchivist, addStorageMeta, removeStorageMeta, sortByStorageMeta, WithStorageMeta,\n} from '@xyo-network/archivist-abstract'\nimport {\n  ArchivistAllQuerySchema,\n  ArchivistClearQuerySchema,\n  ArchivistCommitQuerySchema,\n  ArchivistConfig,\n  ArchivistDeleteQuerySchema,\n  ArchivistInsertQuery,\n  ArchivistInsertQuerySchema,\n  ArchivistModuleEventData,\n  ArchivistNextOptions,\n  ArchivistNextQuerySchema,\n  AttachableArchivistInstance,\n} from '@xyo-network/archivist-model'\nimport { BoundWitness } from '@xyo-network/boundwitness-model'\nimport {\n  AnyConfigSchema, creatableModule, ModuleInstance, ModuleParams,\n} from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport {\n  Payload, PayloadWithMeta, Schema, WithMeta,\n} from '@xyo-network/payload-model'\nimport { LRUCache } from 'lru-cache'\n\nexport type MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config'\nexport const MemoryArchivistConfigSchema: MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config'\n\nexport type MemoryArchivistConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = ArchivistConfig<\n  WithAdditional<\n    {\n      max?: number\n    },\n    TConfig\n  >,\n  TSchema extends Schema ? TSchema : MemoryArchivistConfigSchema | ArchivistConfig['schema']\n>\n\nexport type MemoryArchivistParams<TConfig extends AnyConfigSchema<MemoryArchivistConfig> = AnyConfigSchema<MemoryArchivistConfig>> =\n  ModuleParams<TConfig>\n@creatableModule()\nexport class MemoryArchivist<\n  TParams extends MemoryArchivistParams<AnyConfigSchema<MemoryArchivistConfig>> = MemoryArchivistParams,\n  TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n>\n  extends AbstractArchivist<TParams, TEventData>\n  implements AttachableArchivistInstance, ModuleInstance {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, MemoryArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = MemoryArchivistConfigSchema\n\n  private _cache?: LRUCache<Hash, WithStorageMeta<PayloadWithMeta>>\n  private _dataHashIndex?: LRUCache<Hash, Hash>\n\n  override get queries() {\n    return [\n      ArchivistAllQuerySchema,\n      ArchivistDeleteQuerySchema,\n      ArchivistClearQuerySchema,\n      ArchivistInsertQuerySchema,\n      ArchivistCommitQuerySchema,\n      ArchivistNextQuerySchema,\n      ...super.queries,\n    ]\n  }\n\n  protected get cache() {\n    this._cache = this._cache ?? new LRUCache<Hash, WithStorageMeta<PayloadWithMeta>>({ max: this.max })\n    return this._cache\n  }\n\n  protected get dataHashIndex() {\n    this._dataHashIndex = this._dataHashIndex ?? new LRUCache<Hash, Hash>({ max: this.max })\n    return this._dataHashIndex\n  }\n\n  protected get max() {\n    return this.config?.max ?? 10_000\n  }\n\n  protected override allHandler(): Promisable<PayloadWithMeta[]> {\n    const all = this.cache.dump().map(([, item]) => item.value).filter(exists)\n    return sortByStorageMeta(all).map(payload => removeStorageMeta(payload))\n  }\n\n  protected override clearHandler(): void | Promise<void> {\n    this.cache.clear()\n    this.dataHashIndex.clear()\n    return this.emit('cleared', { mod: this })\n  }\n\n  protected override async commitHandler(): Promise<WithMeta<BoundWitness>[]> {\n    const payloads = assertEx(await this.allHandler(), () => 'Nothing to commit')\n    const settled = await Promise.allSettled(\n      Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n        const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n        const query = await this.bindQuery(queryPayload, payloads)\n        return (await parent?.query(query[0], query[1]))?.[0]\n      }).filter(exists),\n    )\n    await this.clearHandler()\n    return settled.filter(fulfilled).map(result => result.value).filter(exists)\n  }\n\n  protected override deleteHandler(hashes: Hash[]): Promisable<Hash[]> {\n    const deletedHashes: Hash[] = this.cache\n      .dump()\n      .map(([key, item]) => {\n        if (hashes.includes(key) || hashes.includes(item.value.$hash)) {\n          this.cache.delete(key)\n          return key\n        }\n      })\n      .filter(exists)\n    this.rebuildDataHashIndex()\n    return deletedHashes\n  }\n\n  protected override getHandler(hashes: Hash[]): Promisable<PayloadWithMeta[]> {\n    return hashes.map((hash) => {\n      const resolvedHash = this.dataHashIndex.get(hash) ?? hash\n      const result = this.cache.get(resolvedHash)\n      if (resolvedHash !== hash && !result) {\n        throw new Error('Missing referenced payload')\n      }\n      return removeStorageMeta(result)\n    }).filter(exists)\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<PayloadWithMeta[]> {\n    const pairs = await PayloadBuilder.hashPairs(payloads, { stamp: false })\n    const insertedPayloads = pairs.map(([payload, hash], index) => {\n      return this.cache.get(hash) ?? this.insertPayloadIntoCache(payload, hash, index)\n    })\n\n    return removeStorageMeta(insertedPayloads)\n  }\n\n  protected override async nextHandler(options?: ArchivistNextOptions): Promise<PayloadWithMeta[]> {\n    const {\n      limit, offset, order,\n    } = options ?? {}\n    let all = await this.allHandler()\n    if (order === 'desc') {\n      all = all.reverse()\n    }\n    const allPairs = await PayloadBuilder.hashPairs(all)\n    const startIndex = offset ? allPairs.findIndex(([, hash]) => hash === offset) + 1 : 0\n    return allPairs.slice(startIndex, limit ? startIndex + limit : undefined).map(([payload]) => payload)\n  }\n\n  private insertPayloadIntoCache(payload: PayloadWithMeta, hash: Hash, index = 0): WithStorageMeta<PayloadWithMeta> {\n    const withMeta = addStorageMeta(payload, index)\n    this.cache.set(hash, withMeta)\n    this.dataHashIndex.set(withMeta.$hash, hash)\n    return withMeta\n  }\n\n  private rebuildDataHashIndex() {\n    this._dataHashIndex = new LRUCache<Hash, Hash>({ max: this.max })\n    const pairs = this.cache.dump()\n    for (const [hash, payload] of pairs) {\n      this.dataHashIndex.set(payload.value.$hash, hash)\n    }\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,iBAA6B;AACtC,SACEC,mBAAmBC,gBAAgBC,mBAAmBC,yBACjD;AACP,SACEC,yBACAC,2BACAC,4BAEAC,4BAEAC,4BAGAC,gCAEK;AAEP,SACmBC,uBACZ;AACP,SAASC,sBAAsB;AAI/B,SAASC,gBAAgB;;;;;;;;AAGlB,IAAMC,8BAA2D;AAejE,IAAMC,kBAAN,cAIGC,kBAAAA;SAAAA;;;EAER,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE/CK;EACAC;EAER,IAAaC,UAAU;AACrB,WAAO;MACLC;MACAC;MACAC;MACAC;MACAC;MACAC;SACG,MAAMN;;EAEb;EAEA,IAAcO,QAAQ;AACpB,SAAKT,SAAS,KAAKA,UAAU,IAAIU,SAAiD;MAAEC,KAAK,KAAKA;IAAI,CAAA;AAClG,WAAO,KAAKX;EACd;EAEA,IAAcY,gBAAgB;AAC5B,SAAKX,iBAAiB,KAAKA,kBAAkB,IAAIS,SAAqB;MAAEC,KAAK,KAAKA;IAAI,CAAA;AACtF,WAAO,KAAKV;EACd;EAEA,IAAcU,MAAM;AAClB,WAAO,KAAKE,QAAQF,OAAO;EAC7B;EAEmBG,aAA4C;AAC7D,UAAMC,MAAM,KAAKN,MAAMO,KAAI,EAAGC,IAAI,CAAC,CAAA,EAAGC,IAAAA,MAAUA,KAAKC,KAAK,EAAEC,OAAOC,MAAAA;AACnE,WAAOC,kBAAkBP,GAAAA,EAAKE,IAAIM,CAAAA,YAAWC,kBAAkBD,OAAAA,CAAAA;EACjE;EAEmBE,eAAqC;AACtD,SAAKhB,MAAMiB,MAAK;AAChB,SAAKd,cAAcc,MAAK;AACxB,WAAO,KAAKC,KAAK,WAAW;MAAEC,KAAK;IAAK,CAAA;EAC1C;EAEA,MAAyBC,gBAAmD;AAC1E,UAAMC,WAAWC,SAAS,MAAM,KAAKjB,WAAU,GAAI,MAAM,mBAAA;AACzD,UAAMkB,UAAU,MAAMC,QAAQC,WAC5BC,OAAOC,QAAQ,MAAM,KAAKC,iBAAgB,GAAIC,UAAU,CAAA,CAAE,GAAGrB,IAAI,OAAOsB,WAAAA;AACtE,YAAMC,eAAqC;QAAEC,QAAQnC;MAA2B;AAChF,YAAMoC,QAAQ,MAAM,KAAKC,UAAUH,cAAcV,QAAAA;AACjD,cAAQ,MAAMS,QAAQG,MAAMA,MAAM,CAAA,GAAIA,MAAM,CAAA,CAAE,KAAK,CAAA;IACrD,CAAA,EAAGtB,OAAOC,MAAAA,CAAAA;AAEZ,UAAM,KAAKI,aAAY;AACvB,WAAOO,QAAQZ,OAAOwB,SAAAA,EAAW3B,IAAI4B,CAAAA,WAAUA,OAAO1B,KAAK,EAAEC,OAAOC,MAAAA;EACtE;EAEmByB,cAAcC,QAAoC;AACnE,UAAMC,gBAAwB,KAAKvC,MAChCO,KAAI,EACJC,IAAI,CAAC,CAACgC,KAAK/B,IAAAA,MAAK;AACf,UAAI6B,OAAOG,SAASD,GAAAA,KAAQF,OAAOG,SAAShC,KAAKC,MAAMgC,KAAK,GAAG;AAC7D,aAAK1C,MAAM2C,OAAOH,GAAAA;AAClB,eAAOA;MACT;IACF,CAAA,EACC7B,OAAOC,MAAAA;AACV,SAAKgC,qBAAoB;AACzB,WAAOL;EACT;EAEmBM,WAAWP,QAA+C;AAC3E,WAAOA,OAAO9B,IAAI,CAACsC,SAAAA;AACjB,YAAMC,eAAe,KAAK5C,cAAc6C,IAAIF,IAAAA,KAASA;AACrD,YAAMV,SAAS,KAAKpC,MAAMgD,IAAID,YAAAA;AAC9B,UAAIA,iBAAiBD,QAAQ,CAACV,QAAQ;AACpC,cAAM,IAAIa,MAAM,4BAAA;MAClB;AACA,aAAOlC,kBAAkBqB,MAAAA;IAC3B,CAAA,EAAGzB,OAAOC,MAAAA;EACZ;EAEA,MAAyBsC,cAAc7B,UAAiD;AACtF,UAAM8B,QAAQ,MAAMC,eAAeC,UAAUhC,UAAU;MAAEiC,OAAO;IAAM,CAAA;AACtE,UAAMC,mBAAmBJ,MAAM3C,IAAI,CAAC,CAACM,SAASgC,IAAAA,GAAOU,UAAAA;AACnD,aAAO,KAAKxD,MAAMgD,IAAIF,IAAAA,KAAS,KAAKW,uBAAuB3C,SAASgC,MAAMU,KAAAA;IAC5E,CAAA;AAEA,WAAOzC,kBAAkBwC,gBAAAA;EAC3B;EAEA,MAAyBG,YAAYC,SAA4D;AAC/F,UAAM,EACJC,OAAOC,QAAQC,MAAK,IAClBH,WAAW,CAAC;AAChB,QAAIrD,MAAM,MAAM,KAAKD,WAAU;AAC/B,QAAIyD,UAAU,QAAQ;AACpBxD,YAAMA,IAAIyD,QAAO;IACnB;AACA,UAAMC,WAAW,MAAMZ,eAAeC,UAAU/C,GAAAA;AAChD,UAAM2D,aAAaJ,SAASG,SAASE,UAAU,CAAC,CAAA,EAAGpB,IAAAA,MAAUA,SAASe,MAAAA,IAAU,IAAI;AACpF,WAAOG,SAASG,MAAMF,YAAYL,QAAQK,aAAaL,QAAQQ,MAAAA,EAAW5D,IAAI,CAAC,CAACM,OAAAA,MAAaA,OAAAA;EAC/F;EAEQ2C,uBAAuB3C,SAA0BgC,MAAYU,QAAQ,GAAqC;AAChH,UAAMa,WAAWC,eAAexD,SAAS0C,KAAAA;AACzC,SAAKxD,MAAMuE,IAAIzB,MAAMuB,QAAAA;AACrB,SAAKlE,cAAcoE,IAAIF,SAAS3B,OAAOI,IAAAA;AACvC,WAAOuB;EACT;EAEQzB,uBAAuB;AAC7B,SAAKpD,iBAAiB,IAAIS,SAAqB;MAAEC,KAAK,KAAKA;IAAI,CAAA;AAC/D,UAAMiD,QAAQ,KAAKnD,MAAMO,KAAI;AAC7B,eAAW,CAACuC,MAAMhC,OAAAA,KAAYqC,OAAO;AACnC,WAAKhD,cAAcoE,IAAIzD,QAAQJ,MAAMgC,OAAOI,IAAAA;IAC9C;EACF;AACF;;;;","names":["assertEx","exists","fulfilled","AbstractArchivist","addStorageMeta","removeStorageMeta","sortByStorageMeta","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistCommitQuerySchema","ArchivistDeleteQuerySchema","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","creatableModule","PayloadBuilder","LRUCache","MemoryArchivistConfigSchema","MemoryArchivist","AbstractArchivist","configSchemas","defaultConfigSchema","_cache","_dataHashIndex","queries","ArchivistAllQuerySchema","ArchivistDeleteQuerySchema","ArchivistClearQuerySchema","ArchivistInsertQuerySchema","ArchivistCommitQuerySchema","ArchivistNextQuerySchema","cache","LRUCache","max","dataHashIndex","config","allHandler","all","dump","map","item","value","filter","exists","sortByStorageMeta","payload","removeStorageMeta","clearHandler","clear","emit","mod","commitHandler","payloads","assertEx","settled","Promise","allSettled","Object","values","parentArchivists","commit","parent","queryPayload","schema","query","bindQuery","fulfilled","result","deleteHandler","hashes","deletedHashes","key","includes","$hash","delete","rebuildDataHashIndex","getHandler","hash","resolvedHash","get","Error","insertHandler","pairs","PayloadBuilder","hashPairs","stamp","insertedPayloads","index","insertPayloadIntoCache","nextHandler","options","limit","offset","order","reverse","allPairs","startIndex","findIndex","slice","undefined","withMeta","addStorageMeta","set"]}
+{"version":3,"sources":["../../src/MemoryArchivist.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, WithAdditional } from '@xylabs/object'\nimport { fulfilled, Promisable } from '@xylabs/promise'\nimport { AbstractArchivist, WithStorageMeta } from '@xyo-network/archivist-abstract'\nimport {\n  ArchivistAllQuerySchema,\n  ArchivistClearQuerySchema,\n  ArchivistCommitQuerySchema,\n  ArchivistConfig,\n  ArchivistDeleteQuerySchema,\n  ArchivistInsertQuery,\n  ArchivistInsertQuerySchema,\n  ArchivistModuleEventData,\n  ArchivistNextOptions,\n  ArchivistNextQuerySchema,\n  AttachableArchivistInstance,\n} from '@xyo-network/archivist-model'\nimport { BoundWitness } from '@xyo-network/boundwitness-model'\nimport {\n  AnyConfigSchema, creatableModule, ModuleInstance, ModuleParams,\n} from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema } from '@xyo-network/payload-model'\nimport { LRUCache } from 'lru-cache'\n\nexport type MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config'\nexport const MemoryArchivistConfigSchema: MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config'\n\nexport type MemoryArchivistConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = ArchivistConfig<\n  WithAdditional<\n    {\n      max?: number\n    },\n    TConfig\n  >,\n  TSchema extends Schema ? TSchema : MemoryArchivistConfigSchema | ArchivistConfig['schema']\n>\n\nexport type MemoryArchivistParams<TConfig extends AnyConfigSchema<MemoryArchivistConfig> = AnyConfigSchema<MemoryArchivistConfig>> =\n  ModuleParams<TConfig>\n@creatableModule()\nexport class MemoryArchivist<\n  TParams extends MemoryArchivistParams<AnyConfigSchema<MemoryArchivistConfig>> = MemoryArchivistParams,\n  TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n>\n  extends AbstractArchivist<TParams, TEventData>\n  implements AttachableArchivistInstance, ModuleInstance {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, MemoryArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = MemoryArchivistConfigSchema\n\n  private _cache?: LRUCache<Hash, WithStorageMeta<Payload>>\n  private _dataHashIndex?: LRUCache<Hash, Hash>\n\n  override get queries() {\n    return [\n      ArchivistAllQuerySchema,\n      ArchivistDeleteQuerySchema,\n      ArchivistClearQuerySchema,\n      ArchivistInsertQuerySchema,\n      ArchivistCommitQuerySchema,\n      ArchivistNextQuerySchema,\n      ...super.queries,\n    ]\n  }\n\n  protected get cache() {\n    this._cache = this._cache ?? new LRUCache<Hash, WithStorageMeta<Payload>>({ max: this.max })\n    return this._cache\n  }\n\n  protected get dataHashIndex() {\n    this._dataHashIndex = this._dataHashIndex ?? new LRUCache<Hash, Hash>({ max: this.max })\n    return this._dataHashIndex\n  }\n\n  protected get max() {\n    return this.config?.max ?? 10_000\n  }\n\n  protected override allHandler(): Promisable<Payload[]> {\n    const all = this.cache.dump().map(([, item]) => item.value).filter(exists)\n    return MemoryArchivist.sortByStorageMeta(all).map(payload => MemoryArchivist.removeStorageMeta(payload))\n  }\n\n  protected override clearHandler(): void | Promise<void> {\n    this.cache.clear()\n    this.dataHashIndex.clear()\n    return this.emit('cleared', { mod: this })\n  }\n\n  protected override async commitHandler(): Promise<BoundWitness[]> {\n    const payloads = assertEx(await this.allHandler(), () => 'Nothing to commit')\n    const settled = await Promise.allSettled(\n      Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n        const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n        const query = await this.bindQuery(queryPayload, payloads)\n        return (await parent?.query(query[0], query[1]))?.[0]\n      }).filter(exists),\n    )\n    await this.clearHandler()\n    return settled.filter(fulfilled).map(result => result.value).filter(exists)\n  }\n\n  protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {\n    const deletedHashes: Hash[] = (await Promise.all(this.cache\n      .dump()\n      .map(async ([key, item]) => {\n        const itemValueDataHash = await PayloadBuilder.dataHash(item.value)\n        if (hashes.includes(key) || hashes.includes(itemValueDataHash)) {\n          this.cache.delete(key)\n          return key\n        }\n      })))\n      .filter(exists)\n    await this.rebuildDataHashIndex()\n    return deletedHashes\n  }\n\n  protected override getHandler(hashes: Hash[]): Promisable<Payload[]> {\n    return hashes.map((hash) => {\n      const resolvedHash = this.dataHashIndex.get(hash) ?? hash\n      const result = this.cache.get(resolvedHash)\n      if (resolvedHash !== hash && !result) {\n        throw new Error('Missing referenced payload')\n      }\n      return MemoryArchivist.removeStorageMeta(result)\n    }).filter(exists)\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<Payload[]> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    const insertedPayloads = await Promise.all(pairs.map(async ([payload, hash], index) => {\n      return this.cache.get(hash) ?? await this.insertPayloadIntoCache(payload, hash, index)\n    }))\n\n    return MemoryArchivist.removeStorageMeta(insertedPayloads)\n  }\n\n  protected override async nextHandler(options?: ArchivistNextOptions): Promise<Payload[]> {\n    const {\n      limit, offset, order,\n    } = options ?? {}\n    let all = await this.allHandler()\n    if (order === 'desc') {\n      all = all.reverse()\n    }\n    const allPairs = await PayloadBuilder.hashPairs(all)\n    const startIndex = offset ? allPairs.findIndex(([, hash]) => hash === offset) + 1 : 0\n    return allPairs.slice(startIndex, limit ? startIndex + limit : undefined).map(([payload]) => payload)\n  }\n\n  private async insertPayloadIntoCache(payload: Payload, hash: Hash, index = 0): Promise<WithStorageMeta<Payload>> {\n    const dataHash = await PayloadBuilder.dataHash(payload)\n    const withMeta = await MemoryArchivist.addSequencedStorageMeta(payload, index)\n    this.cache.set(hash, withMeta)\n    this.dataHashIndex.set(dataHash, hash)\n    return withMeta\n  }\n\n  private async rebuildDataHashIndex() {\n    this._dataHashIndex = new LRUCache<Hash, Hash>({ max: this.max })\n    const pairs = this.cache.dump()\n    for (const [hash, payload] of pairs) {\n      const dataHash = await PayloadBuilder.dataHash(payload.value)\n      this.dataHashIndex.set(dataHash, hash)\n    }\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,iBAA6B;AACtC,SAASC,yBAA0C;AACnD,SACEC,yBACAC,2BACAC,4BAEAC,4BAEAC,4BAGAC,gCAEK;AAEP,SACmBC,uBACZ;AACP,SAASC,sBAAsB;AAE/B,SAASC,gBAAgB;;;;;;;;AAGlB,IAAMC,8BAA2D;AAejE,IAAMC,kBAAN,MAAMA,yBAIHC,kBAAAA;SAAAA;;;EAER,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE/CK;EACAC;EAER,IAAaC,UAAU;AACrB,WAAO;MACLC;MACAC;MACAC;MACAC;MACAC;MACAC;SACG,MAAMN;;EAEb;EAEA,IAAcO,QAAQ;AACpB,SAAKT,SAAS,KAAKA,UAAU,IAAIU,SAAyC;MAAEC,KAAK,KAAKA;IAAI,CAAA;AAC1F,WAAO,KAAKX;EACd;EAEA,IAAcY,gBAAgB;AAC5B,SAAKX,iBAAiB,KAAKA,kBAAkB,IAAIS,SAAqB;MAAEC,KAAK,KAAKA;IAAI,CAAA;AACtF,WAAO,KAAKV;EACd;EAEA,IAAcU,MAAM;AAClB,WAAO,KAAKE,QAAQF,OAAO;EAC7B;EAEmBG,aAAoC;AACrD,UAAMC,MAAM,KAAKN,MAAMO,KAAI,EAAGC,IAAI,CAAC,CAAA,EAAGC,IAAAA,MAAUA,KAAKC,KAAK,EAAEC,OAAOC,MAAAA;AACnE,WAAOzB,iBAAgB0B,kBAAkBP,GAAAA,EAAKE,IAAIM,CAAAA,YAAW3B,iBAAgB4B,kBAAkBD,OAAAA,CAAAA;EACjG;EAEmBE,eAAqC;AACtD,SAAKhB,MAAMiB,MAAK;AAChB,SAAKd,cAAcc,MAAK;AACxB,WAAO,KAAKC,KAAK,WAAW;MAAEC,KAAK;IAAK,CAAA;EAC1C;EAEA,MAAyBC,gBAAyC;AAChE,UAAMC,WAAWC,SAAS,MAAM,KAAKjB,WAAU,GAAI,MAAM,mBAAA;AACzD,UAAMkB,UAAU,MAAMC,QAAQC,WAC5BC,OAAOC,QAAQ,MAAM,KAAKC,iBAAgB,GAAIC,UAAU,CAAA,CAAE,GAAGrB,IAAI,OAAOsB,WAAAA;AACtE,YAAMC,eAAqC;QAAEC,QAAQnC;MAA2B;AAChF,YAAMoC,QAAQ,MAAM,KAAKC,UAAUH,cAAcV,QAAAA;AACjD,cAAQ,MAAMS,QAAQG,MAAMA,MAAM,CAAA,GAAIA,MAAM,CAAA,CAAE,KAAK,CAAA;IACrD,CAAA,EAAGtB,OAAOC,MAAAA,CAAAA;AAEZ,UAAM,KAAKI,aAAY;AACvB,WAAOO,QAAQZ,OAAOwB,SAAAA,EAAW3B,IAAI4B,CAAAA,WAAUA,OAAO1B,KAAK,EAAEC,OAAOC,MAAAA;EACtE;EAEA,MAAyByB,cAAcC,QAAiC;AACtE,UAAMC,iBAAyB,MAAMf,QAAQlB,IAAI,KAAKN,MACnDO,KAAI,EACJC,IAAI,OAAO,CAACgC,KAAK/B,IAAAA,MAAK;AACrB,YAAMgC,oBAAoB,MAAMC,eAAeC,SAASlC,KAAKC,KAAK;AAClE,UAAI4B,OAAOM,SAASJ,GAAAA,KAAQF,OAAOM,SAASH,iBAAAA,GAAoB;AAC9D,aAAKzC,MAAM6C,OAAOL,GAAAA;AAClB,eAAOA;MACT;IACF,CAAA,CAAA,GACC7B,OAAOC,MAAAA;AACV,UAAM,KAAKkC,qBAAoB;AAC/B,WAAOP;EACT;EAEmBQ,WAAWT,QAAuC;AACnE,WAAOA,OAAO9B,IAAI,CAACwC,SAAAA;AACjB,YAAMC,eAAe,KAAK9C,cAAc+C,IAAIF,IAAAA,KAASA;AACrD,YAAMZ,SAAS,KAAKpC,MAAMkD,IAAID,YAAAA;AAC9B,UAAIA,iBAAiBD,QAAQ,CAACZ,QAAQ;AACpC,cAAM,IAAIe,MAAM,4BAAA;MAClB;AACA,aAAOhE,iBAAgB4B,kBAAkBqB,MAAAA;IAC3C,CAAA,EAAGzB,OAAOC,MAAAA;EACZ;EAEA,MAAyBwC,cAAc/B,UAAyC;AAC9E,UAAMgC,QAAQ,MAAMX,eAAeY,UAAUjC,QAAAA;AAC7C,UAAMkC,mBAAmB,MAAM/B,QAAQlB,IAAI+C,MAAM7C,IAAI,OAAO,CAACM,SAASkC,IAAAA,GAAOQ,UAAAA;AAC3E,aAAO,KAAKxD,MAAMkD,IAAIF,IAAAA,KAAS,MAAM,KAAKS,uBAAuB3C,SAASkC,MAAMQ,KAAAA;IAClF,CAAA,CAAA;AAEA,WAAOrE,iBAAgB4B,kBAAkBwC,gBAAAA;EAC3C;EAEA,MAAyBG,YAAYC,SAAoD;AACvF,UAAM,EACJC,OAAOC,QAAQC,MAAK,IAClBH,WAAW,CAAC;AAChB,QAAIrD,MAAM,MAAM,KAAKD,WAAU;AAC/B,QAAIyD,UAAU,QAAQ;AACpBxD,YAAMA,IAAIyD,QAAO;IACnB;AACA,UAAMC,WAAW,MAAMtB,eAAeY,UAAUhD,GAAAA;AAChD,UAAM2D,aAAaJ,SAASG,SAASE,UAAU,CAAC,CAAA,EAAGlB,IAAAA,MAAUA,SAASa,MAAAA,IAAU,IAAI;AACpF,WAAOG,SAASG,MAAMF,YAAYL,QAAQK,aAAaL,QAAQQ,MAAAA,EAAW5D,IAAI,CAAC,CAACM,OAAAA,MAAaA,OAAAA;EAC/F;EAEA,MAAc2C,uBAAuB3C,SAAkBkC,MAAYQ,QAAQ,GAAsC;AAC/G,UAAMb,WAAW,MAAMD,eAAeC,SAAS7B,OAAAA;AAC/C,UAAMuD,WAAW,MAAMlF,iBAAgBmF,wBAAwBxD,SAAS0C,KAAAA;AACxE,SAAKxD,MAAMuE,IAAIvB,MAAMqB,QAAAA;AACrB,SAAKlE,cAAcoE,IAAI5B,UAAUK,IAAAA;AACjC,WAAOqB;EACT;EAEA,MAAcvB,uBAAuB;AACnC,SAAKtD,iBAAiB,IAAIS,SAAqB;MAAEC,KAAK,KAAKA;IAAI,CAAA;AAC/D,UAAMmD,QAAQ,KAAKrD,MAAMO,KAAI;AAC7B,eAAW,CAACyC,MAAMlC,OAAAA,KAAYuC,OAAO;AACnC,YAAMV,WAAW,MAAMD,eAAeC,SAAS7B,QAAQJ,KAAK;AAC5D,WAAKP,cAAcoE,IAAI5B,UAAUK,IAAAA;IACnC;EACF;AACF;;;;","names":["assertEx","exists","fulfilled","AbstractArchivist","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistCommitQuerySchema","ArchivistDeleteQuerySchema","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","creatableModule","PayloadBuilder","LRUCache","MemoryArchivistConfigSchema","MemoryArchivist","AbstractArchivist","configSchemas","defaultConfigSchema","_cache","_dataHashIndex","queries","ArchivistAllQuerySchema","ArchivistDeleteQuerySchema","ArchivistClearQuerySchema","ArchivistInsertQuerySchema","ArchivistCommitQuerySchema","ArchivistNextQuerySchema","cache","LRUCache","max","dataHashIndex","config","allHandler","all","dump","map","item","value","filter","exists","sortByStorageMeta","payload","removeStorageMeta","clearHandler","clear","emit","mod","commitHandler","payloads","assertEx","settled","Promise","allSettled","Object","values","parentArchivists","commit","parent","queryPayload","schema","query","bindQuery","fulfilled","result","deleteHandler","hashes","deletedHashes","key","itemValueDataHash","PayloadBuilder","dataHash","includes","delete","rebuildDataHashIndex","getHandler","hash","resolvedHash","get","Error","insertHandler","pairs","hashPairs","insertedPayloads","index","insertPayloadIntoCache","nextHandler","options","limit","offset","order","reverse","allPairs","startIndex","findIndex","slice","undefined","withMeta","addSequencedStorageMeta","set"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/archivist-memory",
-  "version": "3.5.1",
+  "version": "3.6.0-rc.1",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -29,28 +29,29 @@
   "module": "dist/neutral/index.mjs",
   "types": "dist/neutral/index.d.ts",
   "dependencies": {
-    "@xylabs/assert": "^4.4.9",
-    "@xylabs/exists": "^4.4.9",
-    "@xylabs/hex": "^4.4.9",
-    "@xylabs/object": "^4.4.9",
-    "@xylabs/promise": "^4.4.9",
-    "@xyo-network/archivist-abstract": "^3.5.1",
-    "@xyo-network/archivist-model": "^3.5.1",
-    "@xyo-network/boundwitness-model": "^3.5.1",
-    "@xyo-network/module-model": "^3.5.1",
-    "@xyo-network/payload-builder": "^3.5.1",
-    "@xyo-network/payload-model": "^3.5.1",
+    "@xylabs/assert": "^4.4.12",
+    "@xylabs/exists": "^4.4.12",
+    "@xylabs/hex": "^4.4.12",
+    "@xylabs/object": "^4.4.12",
+    "@xylabs/promise": "^4.4.12",
+    "@xyo-network/archivist-abstract": "^3.6.0-rc.1",
+    "@xyo-network/archivist-model": "^3.6.0-rc.1",
+    "@xyo-network/boundwitness-model": "^3.6.0-rc.1",
+    "@xyo-network/module-model": "^3.6.0-rc.1",
+    "@xyo-network/payload-builder": "^3.6.0-rc.1",
+    "@xyo-network/payload-model": "^3.6.0-rc.1",
     "lru-cache": "^11.0.2"
   },
   "devDependencies": {
     "@xylabs/ts-scripts-yarn3": "^4.2.4",
     "@xylabs/tsconfig": "^4.2.4",
-    "@xylabs/vitest-extended": "^4.4.9",
-    "@xyo-network/account": "^3.5.1",
+    "@xylabs/vitest-extended": "^4.4.12",
+    "@xyo-network/account": "^3.6.0-rc.1",
     "typescript": "^5.7.2",
-    "vitest": "^2.1.5"
+    "vitest": "^2.1.8"
   },
   "publishConfig": {
     "access": "public"
-  }
+  },
+  "stableVersion": "3.5.2"
 }

package/src/MemoryArchivist.ts

@@ -3,9 +3,7 @@ import { exists } from '@xylabs/exists'
 import { Hash } from '@xylabs/hex'
 import { EmptyObject, WithAdditional } from '@xylabs/object'
 import { fulfilled, Promisable } from '@xylabs/promise'
-import {
-  AbstractArchivist, addStorageMeta, removeStorageMeta, sortByStorageMeta, WithStorageMeta,
-} from '@xyo-network/archivist-abstract'
+import { AbstractArchivist, WithStorageMeta } from '@xyo-network/archivist-abstract'
 import {
   ArchivistAllQuerySchema,
   ArchivistClearQuerySchema,
@@ -24,9 +22,7 @@ import {
   AnyConfigSchema, creatableModule, ModuleInstance, ModuleParams,
 } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
-import {
-  Payload, PayloadWithMeta, Schema, WithMeta,
-} from '@xyo-network/payload-model'
+import { Payload, Schema } from '@xyo-network/payload-model'
 import { LRUCache } from 'lru-cache'
 
 export type MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config'
@@ -54,7 +50,7 @@ export class MemoryArchivist<
   static override readonly configSchemas: Schema[] = [...super.configSchemas, MemoryArchivistConfigSchema]
   static override readonly defaultConfigSchema: Schema = MemoryArchivistConfigSchema
 
-  private _cache?: LRUCache<Hash, WithStorageMeta<PayloadWithMeta>>
+  private _cache?: LRUCache<Hash, WithStorageMeta<Payload>>
   private _dataHashIndex?: LRUCache<Hash, Hash>
 
   override get queries() {
@@ -70,7 +66,7 @@ export class MemoryArchivist<
   }
 
   protected get cache() {
-    this._cache = this._cache ?? new LRUCache<Hash, WithStorageMeta<PayloadWithMeta>>({ max: this.max })
+    this._cache = this._cache ?? new LRUCache<Hash, WithStorageMeta<Payload>>({ max: this.max })
     return this._cache
   }
 
@@ -83,9 +79,9 @@ export class MemoryArchivist<
     return this.config?.max ?? 10_000
   }
 
-  protected override allHandler(): Promisable<PayloadWithMeta[]> {
+  protected override allHandler(): Promisable<Payload[]> {
     const all = this.cache.dump().map(([, item]) => item.value).filter(exists)
-    return sortByStorageMeta(all).map(payload => removeStorageMeta(payload))
+    return MemoryArchivist.sortByStorageMeta(all).map(payload => MemoryArchivist.removeStorageMeta(payload))
   }
 
   protected override clearHandler(): void | Promise<void> {
@@ -94,7 +90,7 @@ export class MemoryArchivist<
     return this.emit('cleared', { mod: this })
   }
 
-  protected override async commitHandler(): Promise<WithMeta<BoundWitness>[]> {
+  protected override async commitHandler(): Promise<BoundWitness[]> {
     const payloads = assertEx(await this.allHandler(), () => 'Nothing to commit')
     const settled = await Promise.allSettled(
       Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {
@@ -107,41 +103,42 @@ export class MemoryArchivist<
     return settled.filter(fulfilled).map(result => result.value).filter(exists)
   }
 
-  protected override deleteHandler(hashes: Hash[]): Promisable<Hash[]> {
-    const deletedHashes: Hash[] = this.cache
+  protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {
+    const deletedHashes: Hash[] = (await Promise.all(this.cache
       .dump()
-      .map(([key, item]) => {
-        if (hashes.includes(key) || hashes.includes(item.value.$hash)) {
+      .map(async ([key, item]) => {
+        const itemValueDataHash = await PayloadBuilder.dataHash(item.value)
+        if (hashes.includes(key) || hashes.includes(itemValueDataHash)) {
           this.cache.delete(key)
           return key
         }
-      })
+      })))
       .filter(exists)
-    this.rebuildDataHashIndex()
+    await this.rebuildDataHashIndex()
     return deletedHashes
   }
 
-  protected override getHandler(hashes: Hash[]): Promisable<PayloadWithMeta[]> {
+  protected override getHandler(hashes: Hash[]): Promisable<Payload[]> {
     return hashes.map((hash) => {
       const resolvedHash = this.dataHashIndex.get(hash) ?? hash
       const result = this.cache.get(resolvedHash)
       if (resolvedHash !== hash && !result) {
         throw new Error('Missing referenced payload')
       }
-      return removeStorageMeta(result)
+      return MemoryArchivist.removeStorageMeta(result)
     }).filter(exists)
   }
 
-  protected override async insertHandler(payloads: Payload[]): Promise<PayloadWithMeta[]> {
-    const pairs = await PayloadBuilder.hashPairs(payloads, { stamp: false })
-    const insertedPayloads = pairs.map(([payload, hash], index) => {
-      return this.cache.get(hash) ?? this.insertPayloadIntoCache(payload, hash, index)
-    })
+  protected override async insertHandler(payloads: Payload[]): Promise<Payload[]> {
+    const pairs = await PayloadBuilder.hashPairs(payloads)
+    const insertedPayloads = await Promise.all(pairs.map(async ([payload, hash], index) => {
+      return this.cache.get(hash) ?? await this.insertPayloadIntoCache(payload, hash, index)
+    }))
 
-    return removeStorageMeta(insertedPayloads)
+    return MemoryArchivist.removeStorageMeta(insertedPayloads)
   }
 
-  protected override async nextHandler(options?: ArchivistNextOptions): Promise<PayloadWithMeta[]> {
+  protected override async nextHandler(options?: ArchivistNextOptions): Promise<Payload[]> {
     const {
       limit, offset, order,
     } = options ?? {}
@@ -154,18 +151,20 @@ export class MemoryArchivist<
     return allPairs.slice(startIndex, limit ? startIndex + limit : undefined).map(([payload]) => payload)
   }
 
-  private insertPayloadIntoCache(payload: PayloadWithMeta, hash: Hash, index = 0): WithStorageMeta<PayloadWithMeta> {
-    const withMeta = addStorageMeta(payload, index)
+  private async insertPayloadIntoCache(payload: Payload, hash: Hash, index = 0): Promise<WithStorageMeta<Payload>> {
+    const dataHash = await PayloadBuilder.dataHash(payload)
+    const withMeta = await MemoryArchivist.addSequencedStorageMeta(payload, index)
     this.cache.set(hash, withMeta)
-    this.dataHashIndex.set(withMeta.$hash, hash)
+    this.dataHashIndex.set(dataHash, hash)
     return withMeta
   }
 
-  private rebuildDataHashIndex() {
+  private async rebuildDataHashIndex() {
     this._dataHashIndex = new LRUCache<Hash, Hash>({ max: this.max })
     const pairs = this.cache.dump()
     for (const [hash, payload] of pairs) {
-      this.dataHashIndex.set(payload.value.$hash, hash)
+      const dataHash = await PayloadBuilder.dataHash(payload.value)
+      this.dataHashIndex.set(dataHash, hash)
     }
   }
 }