@xyo-network/archivist-memory 3.9.6 → 3.9.8

package/dist/neutral/index.d.ts

@@ -21,6 +21,7 @@ declare class MemoryArchivist<TParams extends MemoryArchivistParams<AnyConfigSch
     static readonly defaultConfigSchema: Schema;
     private _cache?;
     private _dataHashIndex?;
+    private _sequenceIndex;
     get queries(): string[];
     protected get cache(): LRUCache<Lowercase<string>, WithStorageMeta<Payload>, unknown>;
     protected get dataHashIndex(): LRUCache<Lowercase<string>, Lowercase<string>, unknown>;
@@ -33,9 +34,10 @@ declare class MemoryArchivist<TParams extends MemoryArchivistParams<AnyConfigSch
     protected deleteHandler(hashes: Hash[]): Promise<Hash[]>;
     protected getHandler(hashes: Hash[]): Promisable<WithStorageMeta<Payload>[]>;
     protected insertHandler(payloads: WithStorageMeta<Payload>[]): WithStorageMeta<Payload>[];
-    protected nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]>;
+    protected nextHandler(options?: ArchivistNextOptions): Promisable<WithStorageMeta<Payload>[]>;
     private insertPayloadIntoCache;
     private rebuildDataHashIndex;
+    private rebuildSequenceIndex;
 }
 
 export { MemoryArchivist, type MemoryArchivistConfig, MemoryArchivistConfigSchema, type MemoryArchivistParams };

package/dist/neutral/index.mjs

@@ -41,6 +41,7 @@ var MemoryArchivistConfigSchema = "network.xyo.archivist.memory.config";
 var MemoryArchivist = class extends AbstractArchivist {
   _cache;
   _dataHashIndex;
+  _sequenceIndex = [];
   get queries() {
     return [
       ArchivistAllQuerySchema,
@@ -76,11 +77,12 @@ var MemoryArchivist = class extends AbstractArchivist {
     return archivist;
   }
   allHandler() {
-    return [...this.cache.values()].filter(exists).sort(PayloadBuilder.compareStorageMeta);
+    return [...this.cache.values()].filter(exists).toSorted(PayloadBuilder.compareStorageMeta);
   }
   clearHandler() {
     this.cache.clear();
-    this.dataHashIndex.clear();
+    this.rebuildDataHashIndex();
+    this.rebuildDataHashIndex();
     return this.emit("cleared", { mod: this });
   }
   async commitHandler() {
@@ -104,6 +106,7 @@ var MemoryArchivist = class extends AbstractArchivist {
       }
     }))).filter(exists);
     this.rebuildDataHashIndex();
+    await this.rebuildSequenceIndex();
     return deletedHashes;
   }
   getHandler(hashes) {
@@ -118,24 +121,24 @@ var MemoryArchivist = class extends AbstractArchivist {
   }
   insertHandler(payloads) {
     const payloadsWithMeta = payloads.toSorted(PayloadBuilder.compareStorageMeta);
+    this._sequenceIndex.push(...payloadsWithMeta);
     return payloadsWithMeta.map((payload) => {
       return this.insertPayloadIntoCache(payload);
     });
   }
-  async nextHandler(options) {
+  nextHandler(options) {
     const {
       limit = 100,
       cursor,
       order,
       open = true
     } = options ?? {};
-    let all = await this.allHandler();
+    let all = this._sequenceIndex;
     if (order === "desc") {
-      all = all.reverse();
+      all = all.toReversed();
     }
     const startIndex = cursor ? MemoryArchivist.findIndexFromCursor(all, cursor) + (open ? 1 : 0) : 0;
-    const result = all.slice(startIndex, startIndex + limit);
-    return result;
+    return all.slice(startIndex, startIndex + limit);
   }
   insertPayloadIntoCache(payload) {
     this.cache.set(payload._hash, payload);
@@ -149,6 +152,9 @@ var MemoryArchivist = class extends AbstractArchivist {
       this.dataHashIndex.set(payload._dataHash, payload._hash);
     }
   }
+  async rebuildSequenceIndex() {
+    this._sequenceIndex = (await this.allHandler()).toSorted(PayloadBuilder.compareStorageMeta);
+  }
 };
 __publicField(MemoryArchivist, "configSchemas", [...__superGet(MemoryArchivist, MemoryArchivist, "configSchemas"), MemoryArchivistConfigSchema]);
 __publicField(MemoryArchivist, "defaultConfigSchema", MemoryArchivistConfigSchema);

package/dist/neutral/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Archivist.ts","../../src/Config.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { Hash, Hex } from '@xylabs/hex'\nimport { fulfilled, Promisable } from '@xylabs/promise'\nimport { Account, AccountInstance } from '@xyo-network/account'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport {\n  ArchivistAllQuerySchema,\n  ArchivistClearQuerySchema,\n  ArchivistCommitQuerySchema,\n  ArchivistDeleteQuerySchema,\n  ArchivistInsertQuery,\n  ArchivistInsertQuerySchema,\n  ArchivistModuleEventData,\n  ArchivistNextOptions,\n  ArchivistNextQuerySchema,\n  ArchivistParams,\n  AttachableArchivistInstance,\n} from '@xyo-network/archivist-model'\nimport { BoundWitness } from '@xyo-network/boundwitness-model'\nimport {\n  AnyConfigSchema, creatableModule, ModuleInstance,\n} from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport {\n  Payload, Schema, WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport { LRUCache } from 'lru-cache'\n\nimport { MemoryArchivistConfig, MemoryArchivistConfigSchema } from './Config.ts'\n\nexport type MemoryArchivistParams<TConfig extends AnyConfigSchema<MemoryArchivistConfig> = AnyConfigSchema<MemoryArchivistConfig>> =\n  ArchivistParams<TConfig>\n@creatableModule()\nexport class MemoryArchivist<\n  TParams extends MemoryArchivistParams<AnyConfigSchema<MemoryArchivistConfig>> = MemoryArchivistParams,\n  TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n>\n  extends AbstractArchivist<TParams, TEventData>\n  implements AttachableArchivistInstance, ModuleInstance {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, MemoryArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = MemoryArchivistConfigSchema\n\n  private _cache?: LRUCache<Hash, WithStorageMeta<Payload>>\n  private _dataHashIndex?: LRUCache<Hash, Hash>\n\n  override get queries() {\n    return [\n      ArchivistAllQuerySchema,\n      ArchivistDeleteQuerySchema,\n      ArchivistClearQuerySchema,\n      ArchivistInsertQuerySchema,\n      ArchivistCommitQuerySchema,\n      ArchivistNextQuerySchema,\n      ...super.queries,\n    ]\n  }\n\n  protected get cache() {\n    this._cache = this._cache ?? new LRUCache<Hash, WithStorageMeta<Payload>>({ max: this.max })\n    return this._cache\n  }\n\n  protected get dataHashIndex() {\n    this._dataHashIndex = this._dataHashIndex ?? new LRUCache<Hash, Hash>({ max: this.max })\n    return this._dataHashIndex\n  }\n\n  protected get max() {\n    return this.config?.max ?? 10_000\n  }\n\n  private static findIndexFromCursor(payloads: WithStorageMeta[], cursor: Hex) {\n    const index = payloads.findIndex(({ _sequence }) => _sequence === cursor)\n    if (index === -1) {\n      return Infinity // move to the end\n    }\n    return index\n  }\n\n  async from(payloads: Payload[], account?: AccountInstance): Promise<MemoryArchivist> {\n    const archivist = await MemoryArchivist.create({ account: account ?? await Account.random() })\n    await archivist.insert(payloads)\n    return archivist\n  }\n\n  protected override allHandler(): Promisable<WithStorageMeta<Payload>[]> {\n    return [...this.cache.values()].filter(exists).sort(PayloadBuilder.compareStorageMeta)\n  }\n\n  protected override clearHandler(): void | Promise<void> {\n    this.cache.clear()\n    this.dataHashIndex.clear()\n    return this.emit('cleared', { mod: this })\n  }\n\n  protected override async commitHandler(): Promise<BoundWitness[]> {\n    const payloads = assertEx(await this.allHandler(), () => 'Nothing to commit')\n    const settled = await Promise.allSettled(\n      Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n        const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n        const query = await this.bindQuery(queryPayload, payloads)\n        return (await parent?.query(query[0], query[1]))?.[0]\n      }).filter(exists),\n    )\n    await this.clearHandler()\n    return settled.filter(fulfilled).map(result => result.value).filter(exists)\n  }\n\n  protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {\n    const deletedHashes: Hash[] = (await Promise.all(this.cache\n      .dump()\n      .map(async ([key, item]) => {\n        const itemValueDataHash = await PayloadBuilder.dataHash(item.value)\n        if (hashes.includes(key) || hashes.includes(itemValueDataHash)) {\n          this.cache.delete(key)\n          return key\n        }\n      })))\n      .filter(exists)\n    this.rebuildDataHashIndex()\n    return deletedHashes\n  }\n\n  protected override getHandler(hashes: Hash[]): Promisable<WithStorageMeta<Payload>[]> {\n    return hashes.map((hash) => {\n      const resolvedHash = this.dataHashIndex.get(hash) ?? hash\n      const result = this.cache.get(resolvedHash)\n      if (resolvedHash !== hash && !result) {\n        throw new Error('Missing referenced payload')\n      }\n      return result\n    }).filter(exists)\n  }\n\n  protected override insertHandler(payloads: WithStorageMeta<Payload>[]): WithStorageMeta<Payload>[] {\n    const payloadsWithMeta = payloads.toSorted(PayloadBuilder.compareStorageMeta)\n    return payloadsWithMeta.map((payload) => {\n      return this.insertPayloadIntoCache(payload)\n    })\n  }\n\n  protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]> {\n    const {\n      limit = 100, cursor, order, open = true,\n    } = options ?? {}\n    let all = await this.allHandler()\n    if (order === 'desc') {\n      all = all.reverse()\n    }\n    const startIndex = cursor\n      ? MemoryArchivist.findIndexFromCursor(all, cursor) + (open ? 1 : 0)\n      : 0\n    const result = all.slice(startIndex, startIndex + limit)\n    return result\n  }\n\n  private insertPayloadIntoCache(payload: WithStorageMeta<Payload>): WithStorageMeta<Payload> {\n    this.cache.set(payload._hash, payload)\n    this.dataHashIndex.set(payload._dataHash, payload._hash)\n    return payload\n  }\n\n  private rebuildDataHashIndex() {\n    this._dataHashIndex = new LRUCache<Hash, Hash>({ max: this.max })\n    const payloads = this.cache.dump().map(([, item]) => item.value)\n    for (const payload of payloads) {\n      this.dataHashIndex.set(payload._dataHash, payload._hash)\n    }\n  }\n}\n","import type { EmptyObject, WithAdditional } from '@xylabs/object'\nimport type { ArchivistConfig } from '@xyo-network/archivist-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\nexport const MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config' as const\nexport type MemoryArchivistConfigSchema = typeof MemoryArchivistConfigSchema\n\nexport type MemoryArchivistConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = ArchivistConfig<\n  WithAdditional<\n    {\n      max?: number\n    },\n    TConfig\n  >,\n  TSchema extends Schema ? TSchema : MemoryArchivistConfigSchema | ArchivistConfig['schema']\n>\n"],"mappings":";;;;;;;;;;;;;;;;;AAAA,SAAS,gBAAgB;AACzB,SAAS,cAAc;AAEvB,SAAS,iBAA6B;AACtC,SAAS,eAAgC;AACzC,SAAS,yBAAyB;AAClC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EAGA;AAAA,OAGK;AAEP;AAAA,EACmB;AAAA,OACZ;AACP,SAAS,sBAAsB;AAI/B,SAAS,gBAAgB;;;ACvBlB,IAAM,8BAA8B;;;AD8BpC,IAAM,kBAAN,cAIG,kBAC+C;AAAA,EAI/C;AAAA,EACA;AAAA,EAER,IAAa,UAAU;AACrB,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,GAAG,MAAM;AAAA,IACX;AAAA,EACF;AAAA,EAEA,IAAc,QAAQ;AACpB,SAAK,SAAS,KAAK,UAAU,IAAI,SAAyC,EAAE,KAAK,KAAK,IAAI,CAAC;AAC3F,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAc,gBAAgB;AAC5B,SAAK,iBAAiB,KAAK,kBAAkB,IAAI,SAAqB,EAAE,KAAK,KAAK,IAAI,CAAC;AACvF,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAc,MAAM;AAClB,WAAO,KAAK,QAAQ,OAAO;AAAA,EAC7B;AAAA,EAEA,OAAe,oBAAoB,UAA6B,QAAa;AAC3E,UAAM,QAAQ,SAAS,UAAU,CAAC,EAAE,UAAU,MAAM,cAAc,MAAM;AACxE,QAAI,UAAU,IAAI;AAChB,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,KAAK,UAAqB,SAAqD;AACnF,UAAM,YAAY,MAAM,gBAAgB,OAAO,EAAE,SAAS,WAAW,MAAM,QAAQ,OAAO,EAAE,CAAC;AAC7F,UAAM,UAAU,OAAO,QAAQ;AAC/B,WAAO;AAAA,EACT;AAAA,EAEmB,aAAqD;AACtE,WAAO,CAAC,GAAG,KAAK,MAAM,OAAO,CAAC,EAAE,OAAO,MAAM,EAAE,KAAK,eAAe,kBAAkB;AAAA,EACvF;AAAA,EAEmB,eAAqC;AACtD,SAAK,MAAM,MAAM;AACjB,SAAK,cAAc,MAAM;AACzB,WAAO,KAAK,KAAK,WAAW,EAAE,KAAK,KAAK,CAAC;AAAA,EAC3C;AAAA,EAEA,MAAyB,gBAAyC;AAChE,UAAM,WAAW,SAAS,MAAM,KAAK,WAAW,GAAG,MAAM,mBAAmB;AAC5E,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,OAAO,QAAQ,MAAM,KAAK,iBAAiB,GAAG,UAAU,CAAC,CAAC,GAAG,IAAI,OAAO,WAAW;AACjF,cAAM,eAAqC,EAAE,QAAQ,2BAA2B;AAChF,cAAM,QAAQ,MAAM,KAAK,UAAU,cAAc,QAAQ;AACzD,gBAAQ,MAAM,QAAQ,MAAM,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC;AAAA,MACtD,CAAC,EAAE,OAAO,MAAM;AAAA,IAClB;AACA,UAAM,KAAK,aAAa;AACxB,WAAO,QAAQ,OAAO,SAAS,EAAE,IAAI,YAAU,OAAO,KAAK,EAAE,OAAO,MAAM;AAAA,EAC5E;AAAA,EAEA,MAAyB,cAAc,QAAiC;AACtE,UAAM,iBAAyB,MAAM,QAAQ,IAAI,KAAK,MACnD,KAAK,EACL,IAAI,OAAO,CAAC,KAAK,IAAI,MAAM;AAC1B,YAAM,oBAAoB,MAAM,eAAe,SAAS,KAAK,KAAK;AAClE,UAAI,OAAO,SAAS,GAAG,KAAK,OAAO,SAAS,iBAAiB,GAAG;AAC9D,aAAK,MAAM,OAAO,GAAG;AACrB,eAAO;AAAA,MACT;AAAA,IACF,CAAC,CAAC,GACD,OAAO,MAAM;AAChB,SAAK,qBAAqB;AAC1B,WAAO;AAAA,EACT;AAAA,EAEmB,WAAW,QAAwD;AACpF,WAAO,OAAO,IAAI,CAAC,SAAS;AAC1B,YAAM,eAAe,KAAK,cAAc,IAAI,IAAI,KAAK;AACrD,YAAM,SAAS,KAAK,MAAM,IAAI,YAAY;AAC1C,UAAI,iBAAiB,QAAQ,CAAC,QAAQ;AACpC,cAAM,IAAI,MAAM,4BAA4B;AAAA,MAC9C;AACA,aAAO;AAAA,IACT,CAAC,EAAE,OAAO,MAAM;AAAA,EAClB;AAAA,EAEmB,cAAc,UAAkE;AACjG,UAAM,mBAAmB,SAAS,SAAS,eAAe,kBAAkB;AAC5E,WAAO,iBAAiB,IAAI,CAAC,YAAY;AACvC,aAAO,KAAK,uBAAuB,OAAO;AAAA,IAC5C,CAAC;AAAA,EACH;AAAA,EAEA,MAAyB,YAAY,SAAqE;AACxG,UAAM;AAAA,MACJ,QAAQ;AAAA,MAAK;AAAA,MAAQ;AAAA,MAAO,OAAO;AAAA,IACrC,IAAI,WAAW,CAAC;AAChB,QAAI,MAAM,MAAM,KAAK,WAAW;AAChC,QAAI,UAAU,QAAQ;AACpB,YAAM,IAAI,QAAQ;AAAA,IACpB;AACA,UAAM,aAAa,SACf,gBAAgB,oBAAoB,KAAK,MAAM,KAAK,OAAO,IAAI,KAC/D;AACJ,UAAM,SAAS,IAAI,MAAM,YAAY,aAAa,KAAK;AACvD,WAAO;AAAA,EACT;AAAA,EAEQ,uBAAuB,SAA6D;AAC1F,SAAK,MAAM,IAAI,QAAQ,OAAO,OAAO;AACrC,SAAK,cAAc,IAAI,QAAQ,WAAW,QAAQ,KAAK;AACvD,WAAO;AAAA,EACT;AAAA,EAEQ,uBAAuB;AAC7B,SAAK,iBAAiB,IAAI,SAAqB,EAAE,KAAK,KAAK,IAAI,CAAC;AAChE,UAAM,WAAW,KAAK,MAAM,KAAK,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,MAAM,KAAK,KAAK;AAC/D,eAAW,WAAW,UAAU;AAC9B,WAAK,cAAc,IAAI,QAAQ,WAAW,QAAQ,KAAK;AAAA,IACzD;AAAA,EACF;AACF;AAlIE,cANW,iBAMc,iBAA0B,CAAC,GAAG,6CAAM,kBAAe,2BAA2B;AACvG,cAPW,iBAOc,uBAA8B;AAP5C,kBAAN;AAAA,EADN,gBAAgB;AAAA,GACJ;","names":[]}
+{"version":3,"sources":["../../src/Archivist.ts","../../src/Config.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { Hash, Hex } from '@xylabs/hex'\nimport { fulfilled, Promisable } from '@xylabs/promise'\nimport { Account, AccountInstance } from '@xyo-network/account'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport {\n  ArchivistAllQuerySchema,\n  ArchivistClearQuerySchema,\n  ArchivistCommitQuerySchema,\n  ArchivistDeleteQuerySchema,\n  ArchivistInsertQuery,\n  ArchivistInsertQuerySchema,\n  ArchivistModuleEventData,\n  ArchivistNextOptions,\n  ArchivistNextQuerySchema,\n  ArchivistParams,\n  AttachableArchivistInstance,\n} from '@xyo-network/archivist-model'\nimport { BoundWitness } from '@xyo-network/boundwitness-model'\nimport {\n  AnyConfigSchema, creatableModule, ModuleInstance,\n} from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport {\n  Payload, Schema, WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport { LRUCache } from 'lru-cache'\n\nimport { MemoryArchivistConfig, MemoryArchivistConfigSchema } from './Config.ts'\n\nexport type MemoryArchivistParams<TConfig extends AnyConfigSchema<MemoryArchivistConfig> = AnyConfigSchema<MemoryArchivistConfig>> =\n  ArchivistParams<TConfig>\n@creatableModule()\nexport class MemoryArchivist<\n  TParams extends MemoryArchivistParams<AnyConfigSchema<MemoryArchivistConfig>> = MemoryArchivistParams,\n  TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n>\n  extends AbstractArchivist<TParams, TEventData>\n  implements AttachableArchivistInstance, ModuleInstance {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, MemoryArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = MemoryArchivistConfigSchema\n\n  private _cache?: LRUCache<Hash, WithStorageMeta<Payload>>\n  private _dataHashIndex?: LRUCache<Hash, Hash>\n  private _sequenceIndex: WithStorageMeta<Payload>[] = []\n\n  override get queries() {\n    return [\n      ArchivistAllQuerySchema,\n      ArchivistDeleteQuerySchema,\n      ArchivistClearQuerySchema,\n      ArchivistInsertQuerySchema,\n      ArchivistCommitQuerySchema,\n      ArchivistNextQuerySchema,\n      ...super.queries,\n    ]\n  }\n\n  protected get cache() {\n    this._cache = this._cache ?? new LRUCache<Hash, WithStorageMeta<Payload>>({ max: this.max })\n    return this._cache\n  }\n\n  protected get dataHashIndex() {\n    this._dataHashIndex = this._dataHashIndex ?? new LRUCache<Hash, Hash>({ max: this.max })\n    return this._dataHashIndex\n  }\n\n  protected get max() {\n    return this.config?.max ?? 10_000\n  }\n\n  private static findIndexFromCursor(payloads: WithStorageMeta[], cursor: Hex) {\n    const index = payloads.findIndex(({ _sequence }) => _sequence === cursor)\n    if (index === -1) {\n      return Infinity // move to the end\n    }\n    return index\n  }\n\n  async from(payloads: Payload[], account?: AccountInstance): Promise<MemoryArchivist> {\n    const archivist = await MemoryArchivist.create({ account: account ?? await Account.random() })\n    await archivist.insert(payloads)\n    return archivist\n  }\n\n  protected override allHandler(): Promisable<WithStorageMeta<Payload>[]> {\n    return [...this.cache.values()].filter(exists).toSorted(PayloadBuilder.compareStorageMeta)\n  }\n\n  protected override clearHandler(): void | Promise<void> {\n    this.cache.clear()\n    this.rebuildDataHashIndex()\n    this.rebuildDataHashIndex()\n    return this.emit('cleared', { mod: this })\n  }\n\n  protected override async commitHandler(): Promise<BoundWitness[]> {\n    const payloads = assertEx(await this.allHandler(), () => 'Nothing to commit')\n    const settled = await Promise.allSettled(\n      Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n        const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n        const query = await this.bindQuery(queryPayload, payloads)\n        return (await parent?.query(query[0], query[1]))?.[0]\n      }).filter(exists),\n    )\n    await this.clearHandler()\n    return settled.filter(fulfilled).map(result => result.value).filter(exists)\n  }\n\n  protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {\n    const deletedHashes: Hash[] = (await Promise.all(this.cache\n      .dump()\n      .map(async ([key, item]) => {\n        const itemValueDataHash = await PayloadBuilder.dataHash(item.value)\n        if (hashes.includes(key) || hashes.includes(itemValueDataHash)) {\n          this.cache.delete(key)\n          return key\n        }\n      })))\n      .filter(exists)\n    this.rebuildDataHashIndex()\n    await this.rebuildSequenceIndex()\n    return deletedHashes\n  }\n\n  protected override getHandler(hashes: Hash[]): Promisable<WithStorageMeta<Payload>[]> {\n    return hashes.map((hash) => {\n      const resolvedHash = this.dataHashIndex.get(hash) ?? hash\n      const result = this.cache.get(resolvedHash)\n      if (resolvedHash !== hash && !result) {\n        throw new Error('Missing referenced payload')\n      }\n      return result\n    }).filter(exists)\n  }\n\n  protected override insertHandler(payloads: WithStorageMeta<Payload>[]): WithStorageMeta<Payload>[] {\n    const payloadsWithMeta = payloads.toSorted(PayloadBuilder.compareStorageMeta)\n    this._sequenceIndex.push(...payloadsWithMeta)\n    return payloadsWithMeta.map((payload) => {\n      return this.insertPayloadIntoCache(payload)\n    })\n  }\n\n  protected override nextHandler(options?: ArchivistNextOptions): Promisable<WithStorageMeta<Payload>[]> {\n    const {\n      limit = 100, cursor, order, open = true,\n    } = options ?? {}\n    let all = this._sequenceIndex\n    if (order === 'desc') {\n      all = all.toReversed()\n    }\n    const startIndex = cursor\n      ? MemoryArchivist.findIndexFromCursor(all, cursor) + (open ? 1 : 0)\n      : 0\n    return all.slice(startIndex, startIndex + limit)\n  }\n\n  private insertPayloadIntoCache(payload: WithStorageMeta<Payload>): WithStorageMeta<Payload> {\n    this.cache.set(payload._hash, payload)\n    this.dataHashIndex.set(payload._dataHash, payload._hash)\n    return payload\n  }\n\n  private rebuildDataHashIndex() {\n    this._dataHashIndex = new LRUCache<Hash, Hash>({ max: this.max })\n    const payloads = this.cache.dump().map(([, item]) => item.value)\n    for (const payload of payloads) {\n      this.dataHashIndex.set(payload._dataHash, payload._hash)\n    }\n  }\n\n  private async rebuildSequenceIndex() {\n    this._sequenceIndex = (await this.allHandler()).toSorted(PayloadBuilder.compareStorageMeta)\n  }\n}\n","import type { EmptyObject, WithAdditional } from '@xylabs/object'\nimport type { ArchivistConfig } from '@xyo-network/archivist-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\nexport const MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config' as const\nexport type MemoryArchivistConfigSchema = typeof MemoryArchivistConfigSchema\n\nexport type MemoryArchivistConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = ArchivistConfig<\n  WithAdditional<\n    {\n      max?: number\n    },\n    TConfig\n  >,\n  TSchema extends Schema ? TSchema : MemoryArchivistConfigSchema | ArchivistConfig['schema']\n>\n"],"mappings":";;;;;;;;;;;;;;;;;AAAA,SAAS,gBAAgB;AACzB,SAAS,cAAc;AAEvB,SAAS,iBAA6B;AACtC,SAAS,eAAgC;AACzC,SAAS,yBAAyB;AAClC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EAGA;AAAA,OAGK;AAEP;AAAA,EACmB;AAAA,OACZ;AACP,SAAS,sBAAsB;AAI/B,SAAS,gBAAgB;;;ACvBlB,IAAM,8BAA8B;;;AD8BpC,IAAM,kBAAN,cAIG,kBAC+C;AAAA,EAI/C;AAAA,EACA;AAAA,EACA,iBAA6C,CAAC;AAAA,EAEtD,IAAa,UAAU;AACrB,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,GAAG,MAAM;AAAA,IACX;AAAA,EACF;AAAA,EAEA,IAAc,QAAQ;AACpB,SAAK,SAAS,KAAK,UAAU,IAAI,SAAyC,EAAE,KAAK,KAAK,IAAI,CAAC;AAC3F,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAc,gBAAgB;AAC5B,SAAK,iBAAiB,KAAK,kBAAkB,IAAI,SAAqB,EAAE,KAAK,KAAK,IAAI,CAAC;AACvF,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAc,MAAM;AAClB,WAAO,KAAK,QAAQ,OAAO;AAAA,EAC7B;AAAA,EAEA,OAAe,oBAAoB,UAA6B,QAAa;AAC3E,UAAM,QAAQ,SAAS,UAAU,CAAC,EAAE,UAAU,MAAM,cAAc,MAAM;AACxE,QAAI,UAAU,IAAI;AAChB,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,KAAK,UAAqB,SAAqD;AACnF,UAAM,YAAY,MAAM,gBAAgB,OAAO,EAAE,SAAS,WAAW,MAAM,QAAQ,OAAO,EAAE,CAAC;AAC7F,UAAM,UAAU,OAAO,QAAQ;AAC/B,WAAO;AAAA,EACT;AAAA,EAEmB,aAAqD;AACtE,WAAO,CAAC,GAAG,KAAK,MAAM,OAAO,CAAC,EAAE,OAAO,MAAM,EAAE,SAAS,eAAe,kBAAkB;AAAA,EAC3F;AAAA,EAEmB,eAAqC;AACtD,SAAK,MAAM,MAAM;AACjB,SAAK,qBAAqB;AAC1B,SAAK,qBAAqB;AAC1B,WAAO,KAAK,KAAK,WAAW,EAAE,KAAK,KAAK,CAAC;AAAA,EAC3C;AAAA,EAEA,MAAyB,gBAAyC;AAChE,UAAM,WAAW,SAAS,MAAM,KAAK,WAAW,GAAG,MAAM,mBAAmB;AAC5E,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,OAAO,QAAQ,MAAM,KAAK,iBAAiB,GAAG,UAAU,CAAC,CAAC,GAAG,IAAI,OAAO,WAAW;AACjF,cAAM,eAAqC,EAAE,QAAQ,2BAA2B;AAChF,cAAM,QAAQ,MAAM,KAAK,UAAU,cAAc,QAAQ;AACzD,gBAAQ,MAAM,QAAQ,MAAM,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC;AAAA,MACtD,CAAC,EAAE,OAAO,MAAM;AAAA,IAClB;AACA,UAAM,KAAK,aAAa;AACxB,WAAO,QAAQ,OAAO,SAAS,EAAE,IAAI,YAAU,OAAO,KAAK,EAAE,OAAO,MAAM;AAAA,EAC5E;AAAA,EAEA,MAAyB,cAAc,QAAiC;AACtE,UAAM,iBAAyB,MAAM,QAAQ,IAAI,KAAK,MACnD,KAAK,EACL,IAAI,OAAO,CAAC,KAAK,IAAI,MAAM;AAC1B,YAAM,oBAAoB,MAAM,eAAe,SAAS,KAAK,KAAK;AAClE,UAAI,OAAO,SAAS,GAAG,KAAK,OAAO,SAAS,iBAAiB,GAAG;AAC9D,aAAK,MAAM,OAAO,GAAG;AACrB,eAAO;AAAA,MACT;AAAA,IACF,CAAC,CAAC,GACD,OAAO,MAAM;AAChB,SAAK,qBAAqB;AAC1B,UAAM,KAAK,qBAAqB;AAChC,WAAO;AAAA,EACT;AAAA,EAEmB,WAAW,QAAwD;AACpF,WAAO,OAAO,IAAI,CAAC,SAAS;AAC1B,YAAM,eAAe,KAAK,cAAc,IAAI,IAAI,KAAK;AACrD,YAAM,SAAS,KAAK,MAAM,IAAI,YAAY;AAC1C,UAAI,iBAAiB,QAAQ,CAAC,QAAQ;AACpC,cAAM,IAAI,MAAM,4BAA4B;AAAA,MAC9C;AACA,aAAO;AAAA,IACT,CAAC,EAAE,OAAO,MAAM;AAAA,EAClB;AAAA,EAEmB,cAAc,UAAkE;AACjG,UAAM,mBAAmB,SAAS,SAAS,eAAe,kBAAkB;AAC5E,SAAK,eAAe,KAAK,GAAG,gBAAgB;AAC5C,WAAO,iBAAiB,IAAI,CAAC,YAAY;AACvC,aAAO,KAAK,uBAAuB,OAAO;AAAA,IAC5C,CAAC;AAAA,EACH;AAAA,EAEmB,YAAY,SAAwE;AACrG,UAAM;AAAA,MACJ,QAAQ;AAAA,MAAK;AAAA,MAAQ;AAAA,MAAO,OAAO;AAAA,IACrC,IAAI,WAAW,CAAC;AAChB,QAAI,MAAM,KAAK;AACf,QAAI,UAAU,QAAQ;AACpB,YAAM,IAAI,WAAW;AAAA,IACvB;AACA,UAAM,aAAa,SACf,gBAAgB,oBAAoB,KAAK,MAAM,KAAK,OAAO,IAAI,KAC/D;AACJ,WAAO,IAAI,MAAM,YAAY,aAAa,KAAK;AAAA,EACjD;AAAA,EAEQ,uBAAuB,SAA6D;AAC1F,SAAK,MAAM,IAAI,QAAQ,OAAO,OAAO;AACrC,SAAK,cAAc,IAAI,QAAQ,WAAW,QAAQ,KAAK;AACvD,WAAO;AAAA,EACT;AAAA,EAEQ,uBAAuB;AAC7B,SAAK,iBAAiB,IAAI,SAAqB,EAAE,KAAK,KAAK,IAAI,CAAC;AAChE,UAAM,WAAW,KAAK,MAAM,KAAK,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,MAAM,KAAK,KAAK;AAC/D,eAAW,WAAW,UAAU;AAC9B,WAAK,cAAc,IAAI,QAAQ,WAAW,QAAQ,KAAK;AAAA,IACzD;AAAA,EACF;AAAA,EAEA,MAAc,uBAAuB;AACnC,SAAK,kBAAkB,MAAM,KAAK,WAAW,GAAG,SAAS,eAAe,kBAAkB;AAAA,EAC5F;AACF;AAzIE,cANW,iBAMc,iBAA0B,CAAC,GAAG,6CAAM,kBAAe,2BAA2B;AACvG,cAPW,iBAOc,uBAA8B;AAP5C,kBAAN;AAAA,EADN,gBAAgB;AAAA,GACJ;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/archivist-memory",
-  "version": "3.9.6",
+  "version": "3.9.8",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -34,13 +34,13 @@
     "@xylabs/hex": "^4.5.1",
     "@xylabs/object": "^4.5.1",
     "@xylabs/promise": "^4.5.1",
-    "@xyo-network/account": "^3.9.6",
-    "@xyo-network/archivist-abstract": "^3.9.6",
-    "@xyo-network/archivist-model": "^3.9.6",
-    "@xyo-network/boundwitness-model": "^3.9.6",
-    "@xyo-network/module-model": "^3.9.6",
-    "@xyo-network/payload-builder": "^3.9.6",
-    "@xyo-network/payload-model": "^3.9.6",
+    "@xyo-network/account": "^3.9.8",
+    "@xyo-network/archivist-abstract": "^3.9.8",
+    "@xyo-network/archivist-model": "^3.9.8",
+    "@xyo-network/boundwitness-model": "^3.9.8",
+    "@xyo-network/module-model": "^3.9.8",
+    "@xyo-network/payload-builder": "^3.9.8",
+    "@xyo-network/payload-model": "^3.9.8",
     "lru-cache": "^11.0.2"
   },
   "devDependencies": {
@@ -48,10 +48,10 @@
     "@xylabs/ts-scripts-yarn3": "^5.0.24",
     "@xylabs/tsconfig": "^5.0.24",
     "@xylabs/vitest-extended": "^4.5.1",
-    "@xyo-network/account": "^3.9.6",
-    "@xyo-network/archivist-acceptance-tests": "^3.9.6",
-    "@xyo-network/id-payload-plugin": "^3.9.6",
-    "@xyo-network/payload-wrapper": "^3.9.6",
+    "@xyo-network/account": "^3.9.8",
+    "@xyo-network/archivist-acceptance-tests": "^3.9.8",
+    "@xyo-network/id-payload-plugin": "^3.9.8",
+    "@xyo-network/payload-wrapper": "^3.9.8",
     "typescript": "^5.7.3",
     "vitest": "^3.0.5"
   },

package/src/Archivist.ts

@@ -43,6 +43,7 @@ export class MemoryArchivist<
 
   private _cache?: LRUCache<Hash, WithStorageMeta<Payload>>
   private _dataHashIndex?: LRUCache<Hash, Hash>
+  private _sequenceIndex: WithStorageMeta<Payload>[] = []
 
   override get queries() {
     return [
@@ -85,12 +86,13 @@ export class MemoryArchivist<
   }
 
   protected override allHandler(): Promisable<WithStorageMeta<Payload>[]> {
-    return [...this.cache.values()].filter(exists).sort(PayloadBuilder.compareStorageMeta)
+    return [...this.cache.values()].filter(exists).toSorted(PayloadBuilder.compareStorageMeta)
   }
 
   protected override clearHandler(): void | Promise<void> {
     this.cache.clear()
-    this.dataHashIndex.clear()
+    this.rebuildDataHashIndex()
+    this.rebuildDataHashIndex()
     return this.emit('cleared', { mod: this })
   }
 
@@ -119,6 +121,7 @@ export class MemoryArchivist<
       })))
       .filter(exists)
     this.rebuildDataHashIndex()
+    await this.rebuildSequenceIndex()
     return deletedHashes
   }
 
@@ -135,24 +138,24 @@ export class MemoryArchivist<
 
   protected override insertHandler(payloads: WithStorageMeta<Payload>[]): WithStorageMeta<Payload>[] {
     const payloadsWithMeta = payloads.toSorted(PayloadBuilder.compareStorageMeta)
+    this._sequenceIndex.push(...payloadsWithMeta)
     return payloadsWithMeta.map((payload) => {
       return this.insertPayloadIntoCache(payload)
     })
   }
 
-  protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]> {
+  protected override nextHandler(options?: ArchivistNextOptions): Promisable<WithStorageMeta<Payload>[]> {
     const {
       limit = 100, cursor, order, open = true,
     } = options ?? {}
-    let all = await this.allHandler()
+    let all = this._sequenceIndex
     if (order === 'desc') {
-      all = all.reverse()
+      all = all.toReversed()
     }
     const startIndex = cursor
       ? MemoryArchivist.findIndexFromCursor(all, cursor) + (open ? 1 : 0)
       : 0
-    const result = all.slice(startIndex, startIndex + limit)
-    return result
+    return all.slice(startIndex, startIndex + limit)
   }
 
   private insertPayloadIntoCache(payload: WithStorageMeta<Payload>): WithStorageMeta<Payload> {
@@ -168,4 +171,8 @@ export class MemoryArchivist<
       this.dataHashIndex.set(payload._dataHash, payload._hash)
     }
   }
+
+  private async rebuildSequenceIndex() {
+    this._sequenceIndex = (await this.allHandler()).toSorted(PayloadBuilder.compareStorageMeta)
+  }
 }