@xyo-network/diviner-payload-generic 4.3.0 → 5.0.0

package/dist/neutral/Diviner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAEnD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,KAAK,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC/F,OAAO,KAAK,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACzF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,KAAK,EACV,KAAK,EACL,oBAAoB,EACpB,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAI3C,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EACf,eAAe,EAChB,MAAM,4BAA4B,CAAA;AAMnC,eAAO,MAAM,iCAAiC,EAAG,4CAAqD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,MAAM,MAAM,2BAA2B,GAAG,oBAAoB,CAC5D;IACE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;CACnB,EACD,iCAAiC,CAClC,CAAA;AAED,qBACa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,CAAC,2BAA2B,CAAC,GAAG,oBAAoB,CAAC,2BAA2B,CAAC,EACrH,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,eAAe,CAAC,OAAO,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,EAChE,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC,CAAK;IACjE,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,CAAK;IAExD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,OAAO,CAAC,CAAK;IACrB,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,OAAO,IAAI,MAAM,EAAE,CAEhC;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,GAAG;cAOxB,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAetE,UAAU;cAQD,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAkCzE,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAGD,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAED,SAAS,CAAC,mBAAmB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAEjF;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS;cASlD,WAAW;IAa3B,OAAO,CAAC,aAAa;CAYtB"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAEnD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,KAAK,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC/F,OAAO,KAAK,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACzF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,KAAK,EACV,KAAK,EACL,oBAAoB,EACpB,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAI3C,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAEf,eAAe,EAChB,MAAM,4BAA4B,CAAA;AAMnC,eAAO,MAAM,iCAAiC,EAAG,4CAAqD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,MAAM,MAAM,2BAA2B,GAAG,oBAAoB,CAC5D;IACE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;CACnB,EACD,iCAAiC,CAClC,CAAA;AAED,qBACa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,CAAC,2BAA2B,CAAC,GAAG,oBAAoB,CAAC,2BAA2B,CAAC,EACrH,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,eAAe,CAAC,OAAO,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,EAChE,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC,CAAK;IACjE,SAAS,CAAC,gBAAgB,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,CAAK;IAExD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,OAAO,CAAC,CAAU;IAC1B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,OAAO,IAAI,MAAM,EAAE,CAEhC;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,GAAG;cAOxB,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAetE,UAAU;cAQD,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAkCzE,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAGD,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAED,SAAS,CAAC,mBAAmB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAEjF;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS;cASlD,WAAW;IAa3B,OAAO,CAAC,aAAa;CAYtB"}

package/dist/neutral/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { filterAs } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport type { EventListener } from '@xylabs/events'\nimport { forget } from '@xylabs/forget'\nimport type { Hex } from '@xylabs/hex'\nimport type { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport type {\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { asPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { creatableModule } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema,\n  WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\n@creatableModule()\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends WithStorageMeta<Payload> = WithStorageMeta<Payload>,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithStorageMeta<TOut>[]> = {}\n  protected payloadsWithMeta: WithStorageMeta<TOut>[] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _cursor?: Hex\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', cursor?: Hex) {\n    const payloads = this.payloadsWithMeta.toSorted(PayloadBuilder.compareStorageMeta)\n    if (order === 'desc') payloads.reverse()\n    const startIndex = (cursor ? (payloads.findIndex(payload => payload._sequence === cursor) ?? -1) : -1) + 1\n    return payloads.slice(startIndex)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._cursor = undefined\n      this.payloadsWithMeta = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<TOut[]> {\n    const filters = filterAs(payloads ?? [], asPayloadDivinerQueryPayload)\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as PayloadDivinerQueryPayload\n    await this.updateIndex()\n\n    const {\n\n      schema, schemas, order, limit, cursor, ...props\n    } = filter\n    let all: TOut[] = this.all(order, cursor)\n    if (all) {\n      if (schemas?.length) all = all.filter(payload => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all\n            = Array.isArray(filter)\n              ? all.filter(payload =>\n                  filter.every((value) => {\n                    const prop = payload?.[property]\n                    // TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                    return Array.isArray(prop) && prop.includes?.(value)\n                  }))\n              : all.filter(payload => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined) {\n    await super.stopHandler()\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n  }\n\n  // index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = await archivist.next({ limit: 100, cursor: this._cursor }) as TOut[]\n      while (newPayloads.length > 0) {\n        this._cursor = newPayloads.at(-1)?._sequence\n        assertEx(this.payloadsWithMeta.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        this.indexPayloads(newPayloads)\n        newPayloads = await archivist.next({ limit: 100, cursor: this._cursor }) as TOut[]\n      }\n    })\n  }\n\n  private indexPayloads(payloads: WithStorageMeta<TOut>[]): Hex {\n    this.payloadsWithMeta.push(...payloads)\n\n    // update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const payload of payloads) {\n        this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(payloads.at(-1), () => 'No payloads to index')._sequence\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAAA,SAAS,gBAAgB;AACzB,SAAS,gBAAgB;AAEzB,SAAS,cAAc;AAIvB,SAAS,sBAAsB;AAO/B,SAAS,oCAAoC;AAC7C,SAAS,uBAAuB;AAChC,SAAS,sBAAsB;AAK/B,SAAS,aAAa;AAEtB,IAAM,2BAA2B;AACjC,IAAM,yBAAyB;AAExB,IAAM,oCAAoC;AAW1C,IAAM,wBAAN,cASG,eAA+C;AAAA,EAI7C,YAAqD,CAAC;AAAA,EACtD,mBAA4C,CAAC;AAAA,EAE/C;AAAA,EACA;AAAA,EACA,2BAA2B,IAAI,MAAM;AAAA,EAE7C,IAAc,iBAAiB;AAC7B,WAAO,KAAK,OAAO,kBAAkB;AAAA,EACvC;AAAA,EAEA,IAAc,UAAoB;AAChC,WAAO,CAAC,UAAU,GAAI,KAAK,OAAO,WAAW,CAAC,CAAE;AAAA,EAClD;AAAA,EAEA,IAAc,eAAe;AAC3B,WAAO,KAAK,OAAO,gBAAgB;AAAA,EACrC;AAAA,EAEU,IAAI,QAAe,QAAQ,QAAc;AACjD,UAAM,WAAW,KAAK,iBAAiB,SAAS,eAAe,kBAAkB;AACjF,QAAI,UAAU,OAAQ,UAAS,QAAQ;AACvC,UAAM,cAAc,SAAU,SAAS,UAAU,aAAW,QAAQ,cAAc,MAAM,KAAK,KAAM,MAAM;AACzG,WAAO,SAAS,MAAM,UAAU;AAAA,EAClC;AAAA,EAIA,MAAyB,kBAAkB,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAK,oBAAoB;AAC5B,YAAM,YAAY,MAAM,MAAM,kBAAkB;AAChD,UAAI,YAAY,CAAC,WAAW;AAC1B,cAAM,IAAI,MAAM,0BAA0B;AAAA,MAC5C;AACA,iBAAW,GAAG,YAAY,KAAK,mBAAmB;AAClD,iBAAW,GAAG,WAAW,KAAK,kBAAkB;AAChD,iBAAW,GAAG,WAAW,KAAK,kBAAkB;AAChD,WAAK,qBAAqB;AAAA,IAC5B;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAgB,aAAa;AAC3B,UAAM,KAAK,yBAAyB,aAAa,MAAM;AACrD,WAAK,UAAU;AACf,WAAK,mBAAmB,CAAC;AACzB,WAAK,YAAY,CAAC;AAAA,IACpB,CAAC;AAAA,EACH;AAAA,EAEA,MAAyB,cAAc,UAAmC;AACxE,UAAM,UAAU,SAAS,YAAY,CAAC,GAAG,4BAA4B;AACrE,aAAS,QAAQ,SAAS,GAAG,MAAM,4DAA4D;AAC/F,UAAM,SAAS,SAAS,QAAQ,MAAM,GAAG,MAAM,kCAAkC;AACjF,UAAM,KAAK,YAAY;AAEvB,UAAM;AAAA,MAEJ;AAAA,MAAQ;AAAA,MAAS;AAAA,MAAO;AAAA,MAAO;AAAA,MAAQ,GAAG;AAAA,IAC5C,IAAI;AACJ,QAAI,MAAc,KAAK,IAAI,OAAO,MAAM;AACxC,QAAI,KAAK;AACP,UAAI,SAAS,OAAQ,OAAM,IAAI,OAAO,aAAW,QAAQ,SAAS,QAAQ,MAAM,CAAC;AACjF,UAAI,OAAO,KAAK,KAAK,EAAE,SAAS,GAAG;AACjC,cAAM,2BAA2B,OAAO,QAAQ,KAAK;AACrD,mBAAW,CAAC,MAAMA,OAAM,KAAK,0BAA0B;AACrD,gBAAM,WAAW;AACjB,gBACI,MAAM,QAAQA,OAAM,IAClB,IAAI,OAAO,aACTA,QAAO,MAAM,CAAC,UAAU;AACtB,kBAAMC,QAAO,UAAU,QAAQ;AAE/B,mBAAO,MAAM,QAAQA,KAAI,KAAKA,MAAK,WAAW,KAAK;AAAA,UACrD,CAAC,CAAC,IACJ,IAAI,OAAO,aAAW,UAAU,QAAQ,MAAMD,OAAM;AAAA,QAC5D;AAAA,MACF;AACA,aAAO,QAAQ,IAAI,MAAM,GAAG,KAAK,IAAI;AAAA,IACvC,OAAO;AACL,YAAM,IAAI,MAAM,kCAAkC;AAAA,IACpD;AAAA,EACF;AAAA,EAEU,qBAAyE,MAAM;AACvF;AAAA,OACG,YAAY;AACX,cAAM,KAAK,WAAW;AACtB,cAAM,KAAK,YAAY;AAAA,MACzB,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA,EAGU,qBAAyE,MAAM;AACvF;AAAA,OACG,YAAY;AACX,cAAM,KAAK,WAAW;AACtB,cAAM,KAAK,YAAY;AAAA,MACzB,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEU,sBAA2E,MAAM;AACzF,WAAO,KAAK,YAAY,CAAC;AAAA,EAC3B;AAAA,EAEA,MAAyB,YAAY,UAA+B;AAClE,UAAM,MAAM,YAAY;AACxB,UAAM,YAAY,MAAM,KAAK,kBAAkB,IAAI;AACnD,cAAU,IAAI,YAAY,KAAK,mBAAmB;AAClD,cAAU,IAAI,WAAW,KAAK,kBAAkB;AAChD,cAAU,IAAI,WAAW,KAAK,kBAAkB;AAAA,EAClD;AAAA;AAAA,EAGA,MAAgB,cAAc;AAC5B,UAAM,KAAK,yBAAyB,aAAa,YAAY;AAC3D,YAAM,YAAY,MAAM,KAAK,kBAAkB,IAAI;AACnD,UAAI,cAAc,MAAM,UAAU,KAAK,EAAE,OAAO,KAAK,QAAQ,KAAK,QAAQ,CAAC;AAC3E,aAAO,YAAY,SAAS,GAAG;AAC7B,aAAK,UAAU,YAAY,GAAG,EAAE,GAAG;AACnC,iBAAS,KAAK,iBAAiB,SAAS,YAAY,UAAU,KAAK,cAAc,MAAM,uBAAuB;AAC9G,aAAK,cAAc,WAAW;AAC9B,sBAAc,MAAM,UAAU,KAAK,EAAE,OAAO,KAAK,QAAQ,KAAK,QAAQ,CAAC;AAAA,MACzE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,cAAc,UAAwC;AAC5D,SAAK,iBAAiB,KAAK,GAAG,QAAQ;AAGtC,eAAW,SAAS,KAAK,WAAW,CAAC,GAAG;AACtC,WAAK,UAAU,KAAK,IAAI,KAAK,UAAU,KAAK,KAAK,CAAC;AAClD,iBAAW,WAAW,UAAU;AAC9B,aAAK,UAAU,KAAK,EAAE,KAAK,OAAO;AAAA,MACpC;AAAA,IACF;AACA,WAAO,SAAS,SAAS,GAAG,EAAE,GAAG,MAAM,sBAAsB,EAAE;AAAA,EACjE;AACF;AAhJE,cAVW,uBAUc,iBAA0B,CAAC,GAAG,yDAAM,kBAAe,iCAAiC;AAC7G,cAXW,uBAWc,uBAA8B;AAX5C,wBAAN;AAAA,EADN,gBAAgB;AAAA,GACJ;","names":["filter","prop"]}
+{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { filterAs } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport type { EventListener } from '@xylabs/events'\nimport { forget } from '@xylabs/forget'\nimport type { Hex } from '@xylabs/hex'\nimport type { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport type {\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { asPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { creatableModule } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema,\n  Sequence,\n  WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\n@creatableModule()\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends WithStorageMeta<Payload> = WithStorageMeta<Payload>,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithStorageMeta<TOut>[]> = {}\n  protected payloadsWithMeta: WithStorageMeta<TOut>[] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _cursor?: Sequence\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', cursor?: Hex) {\n    const payloads = this.payloadsWithMeta.toSorted(PayloadBuilder.compareStorageMeta)\n    if (order === 'desc') payloads.reverse()\n    const startIndex = (cursor ? (payloads.findIndex(payload => payload._sequence === cursor) ?? -1) : -1) + 1\n    return payloads.slice(startIndex)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._cursor = undefined\n      this.payloadsWithMeta = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<TOut[]> {\n    const filters = filterAs(payloads ?? [], asPayloadDivinerQueryPayload)\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as PayloadDivinerQueryPayload\n    await this.updateIndex()\n\n    const {\n\n      schema, schemas, order, limit, cursor, ...props\n    } = filter\n    let all: TOut[] = this.all(order, cursor)\n    if (all) {\n      if (schemas?.length) all = all.filter(payload => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all\n            = Array.isArray(filter)\n              ? all.filter(payload =>\n                  filter.every((value) => {\n                    const prop = payload?.[property]\n                    // TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                    return Array.isArray(prop) && prop.includes?.(value)\n                  }))\n              : all.filter(payload => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined) {\n    await super.stopHandler()\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n  }\n\n  // index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = await archivist.next({ limit: 100, cursor: this._cursor }) as TOut[]\n      while (newPayloads.length > 0) {\n        this._cursor = newPayloads.at(-1)?._sequence\n        assertEx(this.payloadsWithMeta.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        this.indexPayloads(newPayloads)\n        newPayloads = await archivist.next({ limit: 100, cursor: this._cursor }) as TOut[]\n      }\n    })\n  }\n\n  private indexPayloads(payloads: WithStorageMeta<TOut>[]): Hex {\n    this.payloadsWithMeta.push(...payloads)\n\n    // update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const payload of payloads) {\n        this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(payloads.at(-1), () => 'No payloads to index')._sequence\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAAA,SAAS,gBAAgB;AACzB,SAAS,gBAAgB;AAEzB,SAAS,cAAc;AAIvB,SAAS,sBAAsB;AAO/B,SAAS,oCAAoC;AAC7C,SAAS,uBAAuB;AAChC,SAAS,sBAAsB;AAM/B,SAAS,aAAa;AAEtB,IAAM,2BAA2B;AACjC,IAAM,yBAAyB;AAExB,IAAM,oCAAoC;AAW1C,IAAM,wBAAN,cASG,eAA+C;AAAA,EAI7C,YAAqD,CAAC;AAAA,EACtD,mBAA4C,CAAC;AAAA,EAE/C;AAAA,EACA;AAAA,EACA,2BAA2B,IAAI,MAAM;AAAA,EAE7C,IAAc,iBAAiB;AAC7B,WAAO,KAAK,OAAO,kBAAkB;AAAA,EACvC;AAAA,EAEA,IAAc,UAAoB;AAChC,WAAO,CAAC,UAAU,GAAI,KAAK,OAAO,WAAW,CAAC,CAAE;AAAA,EAClD;AAAA,EAEA,IAAc,eAAe;AAC3B,WAAO,KAAK,OAAO,gBAAgB;AAAA,EACrC;AAAA,EAEU,IAAI,QAAe,QAAQ,QAAc;AACjD,UAAM,WAAW,KAAK,iBAAiB,SAAS,eAAe,kBAAkB;AACjF,QAAI,UAAU,OAAQ,UAAS,QAAQ;AACvC,UAAM,cAAc,SAAU,SAAS,UAAU,aAAW,QAAQ,cAAc,MAAM,KAAK,KAAM,MAAM;AACzG,WAAO,SAAS,MAAM,UAAU;AAAA,EAClC;AAAA,EAIA,MAAyB,kBAAkB,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAK,oBAAoB;AAC5B,YAAM,YAAY,MAAM,MAAM,kBAAkB;AAChD,UAAI,YAAY,CAAC,WAAW;AAC1B,cAAM,IAAI,MAAM,0BAA0B;AAAA,MAC5C;AACA,iBAAW,GAAG,YAAY,KAAK,mBAAmB;AAClD,iBAAW,GAAG,WAAW,KAAK,kBAAkB;AAChD,iBAAW,GAAG,WAAW,KAAK,kBAAkB;AAChD,WAAK,qBAAqB;AAAA,IAC5B;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAgB,aAAa;AAC3B,UAAM,KAAK,yBAAyB,aAAa,MAAM;AACrD,WAAK,UAAU;AACf,WAAK,mBAAmB,CAAC;AACzB,WAAK,YAAY,CAAC;AAAA,IACpB,CAAC;AAAA,EACH;AAAA,EAEA,MAAyB,cAAc,UAAmC;AACxE,UAAM,UAAU,SAAS,YAAY,CAAC,GAAG,4BAA4B;AACrE,aAAS,QAAQ,SAAS,GAAG,MAAM,4DAA4D;AAC/F,UAAM,SAAS,SAAS,QAAQ,MAAM,GAAG,MAAM,kCAAkC;AACjF,UAAM,KAAK,YAAY;AAEvB,UAAM;AAAA,MAEJ;AAAA,MAAQ;AAAA,MAAS;AAAA,MAAO;AAAA,MAAO;AAAA,MAAQ,GAAG;AAAA,IAC5C,IAAI;AACJ,QAAI,MAAc,KAAK,IAAI,OAAO,MAAM;AACxC,QAAI,KAAK;AACP,UAAI,SAAS,OAAQ,OAAM,IAAI,OAAO,aAAW,QAAQ,SAAS,QAAQ,MAAM,CAAC;AACjF,UAAI,OAAO,KAAK,KAAK,EAAE,SAAS,GAAG;AACjC,cAAM,2BAA2B,OAAO,QAAQ,KAAK;AACrD,mBAAW,CAAC,MAAMA,OAAM,KAAK,0BAA0B;AACrD,gBAAM,WAAW;AACjB,gBACI,MAAM,QAAQA,OAAM,IAClB,IAAI,OAAO,aACTA,QAAO,MAAM,CAAC,UAAU;AACtB,kBAAMC,QAAO,UAAU,QAAQ;AAE/B,mBAAO,MAAM,QAAQA,KAAI,KAAKA,MAAK,WAAW,KAAK;AAAA,UACrD,CAAC,CAAC,IACJ,IAAI,OAAO,aAAW,UAAU,QAAQ,MAAMD,OAAM;AAAA,QAC5D;AAAA,MACF;AACA,aAAO,QAAQ,IAAI,MAAM,GAAG,KAAK,IAAI;AAAA,IACvC,OAAO;AACL,YAAM,IAAI,MAAM,kCAAkC;AAAA,IACpD;AAAA,EACF;AAAA,EAEU,qBAAyE,MAAM;AACvF;AAAA,OACG,YAAY;AACX,cAAM,KAAK,WAAW;AACtB,cAAM,KAAK,YAAY;AAAA,MACzB,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA,EAGU,qBAAyE,MAAM;AACvF;AAAA,OACG,YAAY;AACX,cAAM,KAAK,WAAW;AACtB,cAAM,KAAK,YAAY;AAAA,MACzB,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEU,sBAA2E,MAAM;AACzF,WAAO,KAAK,YAAY,CAAC;AAAA,EAC3B;AAAA,EAEA,MAAyB,YAAY,UAA+B;AAClE,UAAM,MAAM,YAAY;AACxB,UAAM,YAAY,MAAM,KAAK,kBAAkB,IAAI;AACnD,cAAU,IAAI,YAAY,KAAK,mBAAmB;AAClD,cAAU,IAAI,WAAW,KAAK,kBAAkB;AAChD,cAAU,IAAI,WAAW,KAAK,kBAAkB;AAAA,EAClD;AAAA;AAAA,EAGA,MAAgB,cAAc;AAC5B,UAAM,KAAK,yBAAyB,aAAa,YAAY;AAC3D,YAAM,YAAY,MAAM,KAAK,kBAAkB,IAAI;AACnD,UAAI,cAAc,MAAM,UAAU,KAAK,EAAE,OAAO,KAAK,QAAQ,KAAK,QAAQ,CAAC;AAC3E,aAAO,YAAY,SAAS,GAAG;AAC7B,aAAK,UAAU,YAAY,GAAG,EAAE,GAAG;AACnC,iBAAS,KAAK,iBAAiB,SAAS,YAAY,UAAU,KAAK,cAAc,MAAM,uBAAuB;AAC9G,aAAK,cAAc,WAAW;AAC9B,sBAAc,MAAM,UAAU,KAAK,EAAE,OAAO,KAAK,QAAQ,KAAK,QAAQ,CAAC;AAAA,MACzE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,cAAc,UAAwC;AAC5D,SAAK,iBAAiB,KAAK,GAAG,QAAQ;AAGtC,eAAW,SAAS,KAAK,WAAW,CAAC,GAAG;AACtC,WAAK,UAAU,KAAK,IAAI,KAAK,UAAU,KAAK,KAAK,CAAC;AAClD,iBAAW,WAAW,UAAU;AAC9B,aAAK,UAAU,KAAK,EAAE,KAAK,OAAO;AAAA,MACpC;AAAA,IACF;AACA,WAAO,SAAS,SAAS,GAAG,EAAE,GAAG,MAAM,sBAAsB,EAAE;AAAA,EACjE;AACF;AAhJE,cAVW,uBAUc,iBAA0B,CAAC,GAAG,yDAAM,kBAAe,iCAAiC;AAC7G,cAXW,uBAWc,uBAA8B;AAX5C,wBAAN;AAAA,EADN,gBAAgB;AAAA,GACJ;","names":["filter","prop"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/diviner-payload-generic",
-  "version": "4.3.0",
+  "version": "5.0.0",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -28,30 +28,34 @@
   },
   "module": "dist/neutral/index.mjs",
   "types": "dist/neutral/index.d.ts",
+  "files": [
+    "dist",
+    "src"
+  ],
   "dependencies": {
-    "@xylabs/array": "^4.15.0",
-    "@xylabs/assert": "^4.15.0",
-    "@xylabs/events": "^4.15.0",
-    "@xylabs/forget": "^4.15.0",
-    "@xylabs/hex": "^4.15.0",
-    "@xyo-network/archivist-model": "^4.3.0",
-    "@xyo-network/diviner-model": "^4.3.0",
-    "@xyo-network/diviner-payload-abstract": "^4.3.0",
-    "@xyo-network/diviner-payload-model": "^4.3.0",
-    "@xyo-network/module-model": "^4.3.0",
-    "@xyo-network/payload-builder": "^4.3.0",
-    "@xyo-network/payload-model": "^4.3.0",
+    "@xylabs/array": "^5.0.0",
+    "@xylabs/assert": "^5.0.0",
+    "@xylabs/events": "^5.0.0",
+    "@xylabs/forget": "^5.0.0",
+    "@xylabs/hex": "^5.0.0",
+    "@xyo-network/archivist-model": "^5.0.0",
+    "@xyo-network/diviner-model": "^5.0.0",
+    "@xyo-network/diviner-payload-abstract": "^5.0.0",
+    "@xyo-network/diviner-payload-model": "^5.0.0",
+    "@xyo-network/module-model": "^5.0.0",
+    "@xyo-network/payload-builder": "^5.0.0",
+    "@xyo-network/payload-model": "^5.0.0",
     "async-mutex": "^0.5.0"
   },
   "devDependencies": {
-    "@xylabs/delay": "^4.15.0",
-    "@xylabs/ts-scripts-yarn3": "^7.0.1",
-    "@xylabs/tsconfig": "^7.0.1",
-    "@xylabs/vitest-extended": "^4.15.0",
-    "@xyo-network/archivist-indexeddb": "^4.3.0",
-    "@xyo-network/archivist-memory": "^4.3.0",
-    "@xyo-network/node-memory": "^4.3.0",
-    "@xyo-network/payload-builder": "^4.3.0",
+    "@xylabs/delay": "^5.0.0",
+    "@xylabs/ts-scripts-yarn3": "^7.0.2",
+    "@xylabs/tsconfig": "^7.0.2",
+    "@xylabs/vitest-extended": "^5.0.0",
+    "@xyo-network/archivist-indexeddb": "^5.0.0",
+    "@xyo-network/archivist-memory": "^5.0.0",
+    "@xyo-network/node-memory": "^5.0.0",
+    "@xyo-network/payload-builder": "^5.0.0",
     "fake-indexeddb": "^6.0.1",
     "typescript": "^5.8.3",
     "vitest": "^3.2.4"

package/src/Diviner.ts

@@ -17,6 +17,7 @@ import { creatableModule } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
 import type {
   Payload, Schema,
+  Sequence,
   WithStorageMeta,
 } from '@xyo-network/payload-model'
 import { Mutex } from 'async-mutex'
@@ -52,7 +53,7 @@ export class GenericPayloadDiviner<
   protected payloadsWithMeta: WithStorageMeta<TOut>[] = []
 
   private _archivistInstance?: ArchivistInstance
-  private _cursor?: Hex
+  private _cursor?: Sequence
   private _updatePayloadPairsMutex = new Mutex()
 
   protected get indexBatchSize() {

package/src/spec/Diviner.indexdb.spec.ts

@@ -0,0 +1,274 @@
+import '@xylabs/vitest-extended'
+
+import { delay } from '@xylabs/delay'
+import { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb'
+import type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
+import { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
+import { MemoryNode } from '@xyo-network/node-memory'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import type { Payload, WithStorageMeta } from '@xyo-network/payload-model'
+import {
+  IDBCursor,
+  IDBCursorWithValue,
+  IDBDatabase,
+  IDBFactory,
+  IDBIndex,
+  IDBKeyRange,
+  IDBObjectStore,
+  IDBOpenDBRequest,
+  IDBRequest,
+  IDBTransaction,
+  IDBVersionChangeEvent,
+  indexedDB,
+} from 'fake-indexeddb'
+import {
+  beforeAll, describe, expect, it,
+} from 'vitest'
+
+import { GenericPayloadDiviner, GenericPayloadDivinerConfigSchema } from '../Diviner.ts'
+
+/**
+ * @group module
+ * @group diviner
+ */
+
+describe('GenericPayloadDiviner', () => {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const globalThisAny = globalThis as any
+
+  // Augment window with prototypes to ensure instance of comparisons work
+  globalThisAny.IDBCursor = IDBCursor
+  globalThisAny.IDBCursorWithValue = IDBCursorWithValue
+  globalThisAny.IDBDatabase = IDBDatabase
+  globalThisAny.IDBFactory = IDBFactory
+  globalThisAny.IDBIndex = IDBIndex
+  globalThisAny.IDBKeyRange = IDBKeyRange
+  globalThisAny.IDBObjectStore = IDBObjectStore
+  globalThisAny.IDBOpenDBRequest = IDBOpenDBRequest
+  globalThisAny.IDBRequest = IDBRequest
+  globalThisAny.IDBTransaction = IDBTransaction
+  globalThisAny.IDBVersionChangeEvent = IDBVersionChangeEvent
+  globalThisAny.indexedDB = indexedDB
+
+  let archivist: IndexedDbArchivist
+  let sut: GenericPayloadDiviner
+  let node: MemoryNode
+  let payloadA: Payload<{ schema: string; url: string }>
+  let payloadB: Payload<{ foo: string[]; schema: string }>
+  let payloadC: Payload<{ foo: string[]; schema: string }>
+  let payloadD: Payload<{ foo: string[]; schema: string }>
+  let insertedPayloads: WithStorageMeta<Payload>[]
+
+  beforeAll(async () => {
+    payloadA = {
+      schema: 'network.xyo.test',
+      url: 'https://xyo.network',
+    }
+    payloadB = {
+      foo: ['bar', 'baz'],
+      schema: 'network.xyo.debug',
+    }
+    payloadC = {
+      foo: ['one', 'two'],
+      schema: 'network.xyo.debug',
+    }
+    payloadD = {
+      foo: ['aaa', 'bbb'],
+      schema: 'network.xyo.debug',
+    }
+
+    archivist = await IndexedDbArchivist.create({
+      account: 'random',
+      config: { name: 'test', schema: IndexedDbArchivist.defaultConfigSchema },
+    })
+    const [insertedPayloadA] = await archivist.insert([payloadA])
+    await delay(1)
+    const [insertedPayloadB] = await archivist.insert([payloadB])
+    await delay(1)
+    const [insertedPayloadC] = await archivist.insert([payloadC])
+    await delay(1)
+    const [insertedPayloadD] = await archivist.insert([payloadD])
+    await delay(1)
+    insertedPayloads = [insertedPayloadA, insertedPayloadB, insertedPayloadC, insertedPayloadD]
+    const all = await archivist.all()
+    console.log(all)
+    sut = await GenericPayloadDiviner.create({
+      account: 'random',
+      config: {
+        archivist: archivist.address,
+        schema: GenericPayloadDivinerConfigSchema,
+      },
+    })
+    node = await MemoryNode.create({
+      account: 'random',
+      config: { schema: MemoryNode.defaultConfigSchema },
+    })
+    const modules = [archivist, sut]
+    await node.start()
+    await Promise.all(
+      modules.map(async (mod) => {
+        await node.register(mod)
+        await node.attach(mod.address, true)
+      }),
+    )
+    await sut.start()
+  })
+  describe('with filter for', () => {
+    describe('schema', () => {
+      describe('single', () => {
+        it.each(['network.xyo.test', 'network.xyo.debug'])('only returns payloads of that schema', async (schema) => {
+          const schemas = [schema]
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({ schemas })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBeGreaterThan(0)
+          expect(results.every(result => result.schema === schema)).toBe(true)
+        })
+        it('only return single payload of that schema', async () => {
+          const schemas = ['network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({ limit: 1, schemas })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBe(1)
+          expect(await PayloadBuilder.dataHash(results[0])).toBe(await PayloadBuilder.dataHash(payloadD))
+          expect(results.every(result => result.schema === 'network.xyo.debug')).toBe(true)
+        })
+        it('only return single payload of that schema (desc)', async () => {
+          const schemas = ['network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 1, order: 'desc', schemas,
+            })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBe(1)
+          expect(await PayloadBuilder.dataHash(results[0])).toBe(await PayloadBuilder.dataHash(payloadD))
+          expect(results.every(result => result.schema === 'network.xyo.debug')).toBe(true)
+        })
+        it('only return single payload of that schema (asc)', async () => {
+          const schemas = ['network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 1, order: 'asc', schemas,
+            })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBe(1)
+          expect(await PayloadBuilder.dataHash(results[0])).toBe(await PayloadBuilder.dataHash(payloadB))
+          expect(results.every(result => result.schema === 'network.xyo.debug')).toBe(true)
+        })
+      })
+      describe('multiple', () => {
+        it('only returns payloads of that schema', async () => {
+          const schemas = ['network.xyo.test', 'network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({ schemas })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBeGreaterThan(0)
+          expect(results.every(result => schemas.includes(result.schema))).toBe(true)
+        })
+      })
+      describe('paging', () => {
+        it('test paging with multiple calls (asc)', async () => {
+          const schemas = ['network.xyo.test', 'network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, order: 'asc', schemas,
+            })
+            .build()
+          const results = await sut.divine([query])
+          const resultSequences = results.map(result => result._sequence)
+          expect(PayloadBuilder.omitStorageMeta(results)).toStrictEqual([payloadA, payloadB])
+          expect(results.length).toBe(2)
+          expect(resultSequences[0]).toBe(insertedPayloads[0]._sequence)
+          expect(resultSequences[1]).toBe(insertedPayloads[1]._sequence)
+
+          const cursor = resultSequences[1]
+          const query2 = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, cursor, order: 'asc', schemas,
+            })
+            .build()
+          const results2 = await sut.divine([query2])
+          const resultSequences2 = results2.map(result => result._sequence)
+          expect(results2.length).toBe(2)
+          expect(resultSequences2[0]).toBe(insertedPayloads[2]._sequence)
+          expect(resultSequences2[1]).toBe(insertedPayloads[3]._sequence)
+          const cursor2 = resultSequences2[1]
+
+          const query3 = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, cursor: cursor2, order: 'asc', schemas,
+            })
+            .build()
+          const results3 = await sut.divine([query3])
+          expect(results3).toBeArrayOfSize(0)
+        })
+        it('test paging with multiple calls (desc)', async () => {
+          const schemas = ['network.xyo.test', 'network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, order: 'desc', schemas,
+            })
+            .build()
+          const results = await sut.divine([query])
+          const resultSequences = results.map(result => result._sequence)
+          expect(results.length).toBe(2)
+          expect(resultSequences[0]).toBe(insertedPayloads[3]._sequence)
+          expect(resultSequences[1]).toBe(insertedPayloads[2]._sequence)
+
+          const cursor = resultSequences[1]
+          const query2 = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, cursor, order: 'desc', schemas,
+            })
+            .build()
+          const results2 = await sut.divine([query2])
+          const resultSequences2 = results2.map(result => result._sequence)
+          expect(results2.length).toBe(2)
+          expect(resultSequences2[0]).toBe(insertedPayloads[1]._sequence)
+          expect(resultSequences2[1]).toBe(insertedPayloads[0]._sequence)
+          const cursor2 = resultSequences2[1]
+
+          const query3 = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, cursor: cursor2, order: 'desc', schemas,
+            })
+            .build()
+          const results3 = await sut.divine([query3])
+          expect(results3).toBeArrayOfSize(0)
+        })
+      })
+    })
+    describe('custom field', () => {
+      describe('property', () => {
+        it('only returns payloads with that property', async () => {
+          type WithUrl = { url?: string }
+          const url = payloadA.url
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload<{ url?: string }>>({ schema: PayloadDivinerQuerySchema })
+            .fields({ url })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBeGreaterThan(0)
+          expect(results.every(result => (result as WithUrl)?.url === url)).toBe(true)
+        })
+      })
+      describe('array', () => {
+        const cases: string[][] = [['bar'], ['baz'], ['bar', 'baz']]
+        it.each(cases)('only returns payloads that have an array containing all the values supplied', async (...foo) => {
+          type WithFoo = { foo?: string[] }
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload<{ foo?: string[] }>>({ schema: PayloadDivinerQuerySchema })
+            .fields({ foo })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBeGreaterThan(0)
+          // eslint-disable-next-line max-nested-callbacks
+          expect(results.every(result => foo.every(v => (result as unknown as WithFoo)?.foo?.includes(v)))).toBe(true)
+        })
+      })
+    })
+  })
+})

package/src/spec/Diviner.spec.ts

@@ -0,0 +1,251 @@
+/* eslint-disable max-nested-callbacks */
+
+import '@xylabs/vitest-extended'
+
+import { delay } from '@xylabs/delay'
+import { MemoryArchivist } from '@xyo-network/archivist-memory'
+import type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
+import { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
+import { MemoryNode } from '@xyo-network/node-memory'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import type { Payload, WithStorageMeta } from '@xyo-network/payload-model'
+import {
+  beforeAll, describe, expect, it,
+} from 'vitest'
+
+import { GenericPayloadDiviner, GenericPayloadDivinerConfigSchema } from '../Diviner.ts'
+
+/**
+ * @group module
+ * @group diviner
+ */
+
+describe('GenericPayloadDiviner', () => {
+  let archivist: MemoryArchivist
+  let sut: GenericPayloadDiviner
+  let node: MemoryNode
+  const payloadA: Payload<{ schema: string; url: string }> = {
+    schema: 'network.xyo.test',
+    url: 'https://xyo.network',
+  }
+  const payloadB: Payload<{ foo: string[]; schema: string }> = {
+    foo: ['bar', 'baz'],
+    schema: 'network.xyo.debug',
+  }
+  const payloadC: Payload<{ foo: string[]; schema: string }> = {
+    foo: ['one', 'two'],
+    schema: 'network.xyo.debug',
+  }
+  const payloadD: Payload<{ foo: string[]; schema: string }> = {
+    foo: ['aaa', 'bbb'],
+    schema: 'network.xyo.debug',
+  }
+  let insertedPayloads: WithStorageMeta<Payload>[] = []
+  beforeAll(async () => {
+    archivist = await MemoryArchivist.create({
+      account: 'random',
+      config: { name: 'test', schema: MemoryArchivist.defaultConfigSchema },
+    })
+    for (const payload of [payloadA, payloadB, payloadC, payloadD]) {
+      await delay(2)
+      const [insertedPayload] = await archivist.insert([payload])
+      insertedPayloads.push(insertedPayload)
+    }
+    sut = await GenericPayloadDiviner.create({
+      account: 'random',
+      config: {
+        archivist: archivist.address,
+        schema: GenericPayloadDivinerConfigSchema,
+      },
+    })
+    node = await MemoryNode.create({
+      account: 'random',
+      config: { schema: MemoryNode.defaultConfigSchema },
+    })
+    const modules = [archivist, sut]
+    await node.start()
+    await Promise.all(
+      modules.map(async (mod) => {
+        await node.register(mod)
+        await node.attach(mod.address, true)
+      }),
+    )
+    await sut.start()
+  })
+  describe('with filter for', () => {
+    describe('schema', () => {
+      describe('single', () => {
+        it.each(['network.xyo.test', 'network.xyo.debug'])('only returns payloads of that schema', async (schema) => {
+          const schemas = [schema]
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema }).fields({ schemas }).build()
+          const results = await sut.divine([query])
+          expect(results.length).toBeGreaterThan(0)
+          expect(results.every(result => result.schema === schema)).toBe(true)
+        })
+        it('only return single payload of that schema', async () => {
+          const schemas = ['network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({ limit: 1, schemas })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBe(1)
+          expect(await PayloadBuilder.dataHash(results[0])).toBe(await PayloadBuilder.dataHash(insertedPayloads[3]))
+          expect(results.every(result => result.schema === 'network.xyo.debug')).toBe(true)
+        })
+        it('only return single payload of that schema (desc)', async () => {
+          const schemas = ['network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 1, order: 'desc', schemas,
+            })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBe(1)
+          expect(PayloadBuilder.omitStorageMeta(results[0])).toStrictEqual(payloadD)
+          expect(await PayloadBuilder.dataHash(results[0])).toBe(await PayloadBuilder.dataHash(insertedPayloads[3]))
+          expect(results.every(result => result.schema === 'network.xyo.debug')).toBe(true)
+        })
+        it('only return single payload of that schema (asc)', async () => {
+          const schemas = ['network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 1, order: 'asc', schemas,
+            })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBe(1)
+          expect(PayloadBuilder.omitStorageMeta(results[0])).toStrictEqual(payloadB)
+          expect(await PayloadBuilder.dataHash(results[0])).toBe(await PayloadBuilder.dataHash(payloadB))
+          expect(results.every(result => result.schema === 'network.xyo.debug')).toBe(true)
+        })
+      })
+      describe('multiple', () => {
+        it('only returns payloads of that schema', async () => {
+          const schemas = ['network.xyo.test', 'network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({ schemas })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBeGreaterThan(0)
+          expect(results.every(result => schemas.includes(result.schema))).toBe(true)
+        })
+      })
+      describe('paging', () => {
+        it('test paging with multiple calls (asc)', async () => {
+          const schemas = ['network.xyo.test', 'network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, order: 'asc', schemas,
+            })
+            .build()
+          const results = await sut.divine([query])
+          console.warn('results', results)
+          const resultSequences = results.map(result => result._sequence)
+          expect(PayloadBuilder.omitStorageMeta(results)).toStrictEqual([payloadA, payloadB])
+          expect(results.length).toBe(2)
+          expect(resultSequences[0]).toBe(insertedPayloads[0]._sequence)
+          expect(resultSequences[1]).toBe(insertedPayloads[1]._sequence)
+
+          const cursor = resultSequences[1]
+          const query2 = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, cursor, order: 'asc', schemas,
+            })
+            .build()
+          const results2 = await sut.divine([query2])
+          const resultSequences2 = results2.map(result => result._sequence)
+          expect(results2.length).toBe(2)
+          expect(resultSequences2[0]).toBe(insertedPayloads[2]._sequence)
+          expect(resultSequences2[1]).toBe(insertedPayloads[3]._sequence)
+          const cursor2 = resultSequences2[1]
+
+          const query3 = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, cursor: cursor2, order: 'asc', schemas,
+            })
+            .build()
+          const results3 = await sut.divine([query3])
+          expect(results3).toBeArrayOfSize(0)
+        })
+        it('test paging with multiple calls (desc)', async () => {
+          const schemas = ['network.xyo.test', 'network.xyo.debug']
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, order: 'desc', schemas,
+            })
+            .build()
+          const results = await sut.divine([query])
+          const resultSequences = results.map(result => result._sequence)
+          expect(results.length).toBe(2)
+          expect(resultSequences[0]).toBe(insertedPayloads[3]._sequence)
+          expect(resultSequences[1]).toBe(insertedPayloads[2]._sequence)
+
+          const cursor = resultSequences[1]
+          const query2 = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, cursor, order: 'desc', schemas,
+            })
+            .build()
+          const results2 = await sut.divine([query2])
+          const resultSequences2 = results2.map(result => result._sequence)
+          expect(results2.length).toBe(2)
+          expect(resultSequences2[0]).toBe(insertedPayloads[1]._sequence)
+          expect(resultSequences2[1]).toBe(insertedPayloads[0]._sequence)
+          const cursor2 = resultSequences2[1]
+
+          const query3 = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+            .fields({
+              limit: 2, cursor: cursor2, order: 'desc', schemas,
+            })
+            .build()
+          const results3 = await sut.divine([query3])
+          expect(results3).toBeArrayOfSize(0)
+        })
+      })
+    })
+    describe('custom field', () => {
+      describe('property', () => {
+        it('only returns payloads with that property', async () => {
+          type WithUrl = { url?: string }
+          const url = payloadA.url
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload & WithUrl>({ schema: PayloadDivinerQuerySchema })
+            .fields({ url })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBeGreaterThan(0)
+          expect(results.every(result => (result as WithUrl)?.url === url)).toBe(true)
+        })
+      })
+      describe('array', () => {
+        const cases: string[][] = [['bar'], ['baz'], ['bar', 'baz']]
+        it.each(cases)('only returns payloads that have an array containing all the values supplied', async (...foo) => {
+          type WithFoo = { foo?: string[] }
+          const query = new PayloadBuilder<PayloadDivinerQueryPayload & WithFoo>({ schema: PayloadDivinerQuerySchema })
+            .fields({ foo })
+            .build()
+          const results = await sut.divine([query])
+          expect(results.length).toBeGreaterThan(0)
+          expect(results.every(result => foo.every(v => (result as unknown as WithFoo)?.foo?.includes(v)))).toBe(true)
+        })
+      })
+    })
+  })
+  describe('on events', () => {
+    it ('should listen to clear event', async () => {
+      // clear the memory archivist before we test
+      await archivist.clear()
+      // insert a single payload
+      const payload: Payload = { schema: 'network.xyo.test' }
+      await archivist.insert([payload])
+      // build query for a single schema
+      const query = new PayloadBuilder<PayloadDivinerQueryPayload>({ schema: PayloadDivinerQuerySchema })
+        .fields({ schemas: ['network.xyo.test'] })
+        .build()
+      // divine the payload
+      const results = await sut.divine([query])
+      // should be one but has two because of some payload inserted before cleared the archivist
+      // but not the diviner cache
+      expect(results).toBeArrayOfSize(1)
+    })
+  })
+})

package/xy.config.ts

@@ -1,10 +0,0 @@
-import type { XyTsupConfig } from '@xylabs/ts-scripts-yarn3'
-const config: XyTsupConfig = {
-  compile: {
-    browser: {},
-    neutral: { src: true },
-    node: {},
-  },
-}
-
-export default config