@xyo-network/diviner-payload-generic 2.95.4

package/dist/node/Diviner.d.mts

@@ -0,0 +1,27 @@
+import { Hash } from '@xylabs/hex';
+import { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model';
+import { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model';
+import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract';
+import { Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model';
+import { EventListener } from '@xyo-network/module-events';
+import { Payload, WithMeta } from '@xyo-network/payload-model';
+export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams = PayloadDivinerParams, TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut>> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {
+    static configSchemas: "network.xyo.diviner.payload.config"[];
+    protected payloadPairs: [WithMeta<TOut>, Hash][];
+    private _archivistInstance?;
+    private _updatePayloadPairsMutex;
+    protected get indexBatchSize(): number;
+    protected get maxIndexSize(): number;
+    protected get payloadPairsOffset(): Lowercase<string> | undefined;
+    protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
+    protected allAsc(offset?: Hash): WithMeta<TOut>[];
+    protected allDesc(offset?: Hash): WithMeta<TOut>[];
+    protected archivistInstance(): Promise<ArchivistInstance | undefined>;
+    protected archivistInstance(required: true): Promise<ArchivistInstance>;
+    protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
+    protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']>;
+    protected stopHandler(_timeout?: number | undefined): Promise<boolean>;
+    protected updateIndex(): Promise<void>;
+    private indexPayloads;
+}
+//# sourceMappingURL=Diviner.d.ts.map

package/dist/node/Diviner.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}

package/dist/node/Diviner.d.ts

@@ -0,0 +1,27 @@
+import { Hash } from '@xylabs/hex';
+import { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model';
+import { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model';
+import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract';
+import { Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model';
+import { EventListener } from '@xyo-network/module-events';
+import { Payload, WithMeta } from '@xyo-network/payload-model';
+export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams = PayloadDivinerParams, TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut>> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {
+    static configSchemas: "network.xyo.diviner.payload.config"[];
+    protected payloadPairs: [WithMeta<TOut>, Hash][];
+    private _archivistInstance?;
+    private _updatePayloadPairsMutex;
+    protected get indexBatchSize(): number;
+    protected get maxIndexSize(): number;
+    protected get payloadPairsOffset(): Lowercase<string> | undefined;
+    protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
+    protected allAsc(offset?: Hash): WithMeta<TOut>[];
+    protected allDesc(offset?: Hash): WithMeta<TOut>[];
+    protected archivistInstance(): Promise<ArchivistInstance | undefined>;
+    protected archivistInstance(required: true): Promise<ArchivistInstance>;
+    protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
+    protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']>;
+    protected stopHandler(_timeout?: number | undefined): Promise<boolean>;
+    protected updateIndex(): Promise<void>;
+    private indexPayloads;
+}
+//# sourceMappingURL=Diviner.d.ts.map

package/dist/node/Diviner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}

package/dist/node/index.cjs

@@ -0,0 +1,146 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var __publicField = (obj, key, value) => {
+  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
+  return value;
+};
+
+// src/index.ts
+var src_exports = {};
+__export(src_exports, {
+  GenericPayloadDiviner: () => GenericPayloadDiviner
+});
+module.exports = __toCommonJS(src_exports);
+
+// src/Diviner.ts
+var import_assert = require("@xylabs/assert");
+var import_forget = require("@xylabs/forget");
+var import_diviner_payload_abstract = require("@xyo-network/diviner-payload-abstract");
+var import_diviner_payload_model = require("@xyo-network/diviner-payload-model");
+var import_payload_builder = require("@xyo-network/payload-builder");
+var import_async_mutex = require("async-mutex");
+var DEFAULT_INDEX_BATCH_SIZE = 100;
+var DEFAULT_MAX_INDEX_SIZE = 8e3;
+var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner_payload_abstract.PayloadDiviner {
+  payloadPairs = [];
+  _archivistInstance;
+  _updatePayloadPairsMutex = new import_async_mutex.Mutex();
+  get indexBatchSize() {
+    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE;
+  }
+  get maxIndexSize() {
+    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE;
+  }
+  get payloadPairsOffset() {
+    var _a;
+    return (_a = this.payloadPairs.at(-1)) == null ? void 0 : _a[1];
+  }
+  all(order = "asc", offset) {
+    return order === "asc" ? this.allAsc(offset) : this.allDesc(offset);
+  }
+  allAsc(offset) {
+    const pairs = this.payloadPairs;
+    const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
+    return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
+  }
+  allDesc(offset) {
+    const pairs = this.payloadPairs.reverse();
+    const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
+    return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
+  }
+  async archivistInstance(required = false) {
+    if (!this._archivistInstance) {
+      const archivist = await super.archivistInstance();
+      if (required && !archivist) {
+        throw new Error("Failed to find archivist");
+      }
+      archivist == null ? void 0 : archivist.on("inserted", this.onArchivistInsert);
+      this._archivistInstance = archivist;
+    }
+    return this._archivistInstance;
+  }
+  async divineHandler(payloads) {
+    const filters = (payloads == null ? void 0 : payloads.filter(import_diviner_payload_model.isPayloadDivinerQueryPayload)) ?? [];
+    (0, import_assert.assertEx)(filters.length < 2, () => "Multiple PayloadDivinerQuery payloads may not be specified");
+    const filter = (0, import_assert.assertEx)(filters.shift(), () => "No PayloadDivinerQuery specified");
+    await this.updateIndex();
+    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter;
+    let all = this.all(order, offset);
+    if (all) {
+      if (schemas == null ? void 0 : schemas.length)
+        all = all.filter((payload) => schemas.includes(payload.schema));
+      if (Object.keys(props).length > 0) {
+        const additionalFilterCriteria = Object.entries(props);
+        for (const [prop, filter2] of additionalFilterCriteria) {
+          const property = prop;
+          all = Array.isArray(filter2) ? all.filter((payload) => filter2.every((value) => {
+            var _a;
+            const prop2 = payload == null ? void 0 : payload[property];
+            return Array.isArray(prop2) && ((_a = prop2.includes) == null ? void 0 : _a.call(prop2, value));
+          })) : all.filter((payload) => (payload == null ? void 0 : payload[property]) === filter2);
+        }
+      }
+      return limit ? all.slice(0, limit) : all;
+    } else {
+      throw new Error('Archivist does not support "all"');
+    }
+  }
+  onArchivistInsert = ({ payloads }) => {
+    (0, import_forget.forget)(this.indexPayloads(payloads));
+  };
+  async stopHandler(_timeout) {
+    const archivist = await this.archivistInstance(true);
+    archivist.off("inserted", this.onArchivistInsert);
+    return await super.stopHandler();
+  }
+  //index any new payloads
+  async updateIndex() {
+    await this._updatePayloadPairsMutex.runExclusive(async () => {
+      const archivist = await this.archivistInstance(true);
+      let newPayloads = await archivist.next({
+        limit: 100,
+        offset: this.payloadPairsOffset
+      });
+      while (newPayloads.length > 0) {
+        (0, import_assert.assertEx)(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
+        await this.indexPayloads(newPayloads);
+        newPayloads = await archivist.next({
+          limit: 100,
+          offset: this.payloadPairsOffset
+        });
+      }
+    });
+  }
+  async indexPayloads(payloads) {
+    const pairs = await import_payload_builder.PayloadBuilder.hashPairs(payloads);
+    this.payloadPairs.push(...pairs);
+  }
+};
+__name(_GenericPayloadDiviner, "GenericPayloadDiviner");
+__publicField(_GenericPayloadDiviner, "configSchemas", [
+  import_diviner_payload_model.PayloadDivinerConfigSchema
+]);
+var GenericPayloadDiviner = _GenericPayloadDiviner;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  GenericPayloadDiviner
+});
+//# sourceMappingURL=index.cjs.map

package/dist/node/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfigSchema,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams = PayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override configSchemas = [PayloadDivinerConfigSchema]\n\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected get payloadPairsOffset() {\n    return this.payloadPairs.at(-1)?.[1]\n  }\n\n  protected all(order: Order = 'asc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = this.payloadPairs.reverse()\n    const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInsert)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {\n    forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInsert)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: WithMeta<TOut>[]) {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,yBAAN,MAAMA,+BASHC,+CAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBR;EACvC;EAEA,IAAcU,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBT;EACrC;EAEA,IAAcU,qBAAqB;AA/CrC;AAgDI,YAAO,UAAKP,aAAaQ,GAAG,EAAC,MAAtB,mBAA2B;EACpC;EAEUC,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKd,aAAaqB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAK1B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAyB2B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,+DAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAtG9B;AAuGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,CAAC,EAAEE,SAAQ,MAAE;AAC9FyB,8BAAO,KAAKC,cAAc1B,QAAAA,CAAAA;EAC5B;EAEA,MAAyB2B,YAAYC,UAAiD;AACpF,UAAMjC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUkC,IAAI,YAAY,KAAK/B,iBAAiB;AAChD,WAAO,MAAM,MAAM6B,YAAAA;EACrB;;EAGA,MAAgBpB,cAAc;AAC5B,UAAM,KAAKlC,yBAAyByD,aAAa,YAAA;AAC/C,YAAMnC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIsC,cAAe,MAAMpC,UAAUqC,KAAK;QAAEpB,OAAO;QAAK9B,QAAQ,KAAKJ;MAAmB,CAAA;AACtF,aAAOqD,YAAY1B,SAAS,GAAG;AAC7BD,oCAAS,KAAKjC,aAAakC,SAAS0B,YAAY1B,UAAU,KAAK5B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKiD,cAAcK,WAAAA;AACzBA,sBAAe,MAAMpC,UAAUqC,KAAK;UAAEpB,OAAO;UAAK9B,QAAQ,KAAKJ;QAAmB,CAAA;MACpF;IACF,CAAA;EACF;EAEA,MAAcgD,cAAc1B,UAA4B;AACtD,UAAMf,QAAQ,MAAMgD,sCAAeC,UAAUlC,QAAAA;AAC7C,SAAK7B,aAAagE,KAAI,GAAIlD,KAAAA;EAC5B;AACF;AAjHUf;AACR,cAVWD,wBAUKmE,iBAAgB;EAACC;;AAV5B,IAAMpE,wBAAN;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","payloadPairsOffset","at","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","indexPayloads","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}

package/dist/node/index.d.cts

@@ -0,0 +1,2 @@
+export * from './Diviner';
+//# sourceMappingURL=index.d.ts.map

package/dist/node/index.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAA"}

package/dist/node/index.d.mts

@@ -0,0 +1,2 @@
+export * from './Diviner';
+//# sourceMappingURL=index.d.ts.map

package/dist/node/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAA"}

package/dist/node/index.d.ts

@@ -0,0 +1,2 @@
+export * from './Diviner';
+//# sourceMappingURL=index.d.ts.map

package/dist/node/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAA"}

package/dist/node/index.js

@@ -0,0 +1,121 @@
+var __defProp = Object.defineProperty;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __publicField = (obj, key, value) => {
+  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
+  return value;
+};
+
+// src/Diviner.ts
+import { assertEx } from "@xylabs/assert";
+import { forget } from "@xylabs/forget";
+import { PayloadDiviner } from "@xyo-network/diviner-payload-abstract";
+import { isPayloadDivinerQueryPayload, PayloadDivinerConfigSchema } from "@xyo-network/diviner-payload-model";
+import { PayloadBuilder } from "@xyo-network/payload-builder";
+import { Mutex } from "async-mutex";
+var DEFAULT_INDEX_BATCH_SIZE = 100;
+var DEFAULT_MAX_INDEX_SIZE = 8e3;
+var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner {
+  payloadPairs = [];
+  _archivistInstance;
+  _updatePayloadPairsMutex = new Mutex();
+  get indexBatchSize() {
+    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE;
+  }
+  get maxIndexSize() {
+    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE;
+  }
+  get payloadPairsOffset() {
+    var _a;
+    return (_a = this.payloadPairs.at(-1)) == null ? void 0 : _a[1];
+  }
+  all(order = "asc", offset) {
+    return order === "asc" ? this.allAsc(offset) : this.allDesc(offset);
+  }
+  allAsc(offset) {
+    const pairs = this.payloadPairs;
+    const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
+    return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
+  }
+  allDesc(offset) {
+    const pairs = this.payloadPairs.reverse();
+    const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
+    return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
+  }
+  async archivistInstance(required = false) {
+    if (!this._archivistInstance) {
+      const archivist = await super.archivistInstance();
+      if (required && !archivist) {
+        throw new Error("Failed to find archivist");
+      }
+      archivist == null ? void 0 : archivist.on("inserted", this.onArchivistInsert);
+      this._archivistInstance = archivist;
+    }
+    return this._archivistInstance;
+  }
+  async divineHandler(payloads) {
+    const filters = (payloads == null ? void 0 : payloads.filter(isPayloadDivinerQueryPayload)) ?? [];
+    assertEx(filters.length < 2, () => "Multiple PayloadDivinerQuery payloads may not be specified");
+    const filter = assertEx(filters.shift(), () => "No PayloadDivinerQuery specified");
+    await this.updateIndex();
+    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter;
+    let all = this.all(order, offset);
+    if (all) {
+      if (schemas == null ? void 0 : schemas.length)
+        all = all.filter((payload) => schemas.includes(payload.schema));
+      if (Object.keys(props).length > 0) {
+        const additionalFilterCriteria = Object.entries(props);
+        for (const [prop, filter2] of additionalFilterCriteria) {
+          const property = prop;
+          all = Array.isArray(filter2) ? all.filter((payload) => filter2.every((value) => {
+            var _a;
+            const prop2 = payload == null ? void 0 : payload[property];
+            return Array.isArray(prop2) && ((_a = prop2.includes) == null ? void 0 : _a.call(prop2, value));
+          })) : all.filter((payload) => (payload == null ? void 0 : payload[property]) === filter2);
+        }
+      }
+      return limit ? all.slice(0, limit) : all;
+    } else {
+      throw new Error('Archivist does not support "all"');
+    }
+  }
+  onArchivistInsert = ({ payloads }) => {
+    forget(this.indexPayloads(payloads));
+  };
+  async stopHandler(_timeout) {
+    const archivist = await this.archivistInstance(true);
+    archivist.off("inserted", this.onArchivistInsert);
+    return await super.stopHandler();
+  }
+  //index any new payloads
+  async updateIndex() {
+    await this._updatePayloadPairsMutex.runExclusive(async () => {
+      const archivist = await this.archivistInstance(true);
+      let newPayloads = await archivist.next({
+        limit: 100,
+        offset: this.payloadPairsOffset
+      });
+      while (newPayloads.length > 0) {
+        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
+        await this.indexPayloads(newPayloads);
+        newPayloads = await archivist.next({
+          limit: 100,
+          offset: this.payloadPairsOffset
+        });
+      }
+    });
+  }
+  async indexPayloads(payloads) {
+    const pairs = await PayloadBuilder.hashPairs(payloads);
+    this.payloadPairs.push(...pairs);
+  }
+};
+__name(_GenericPayloadDiviner, "GenericPayloadDiviner");
+__publicField(_GenericPayloadDiviner, "configSchemas", [
+  PayloadDivinerConfigSchema
+]);
+var GenericPayloadDiviner = _GenericPayloadDiviner;
+export {
+  GenericPayloadDiviner
+};
+//# sourceMappingURL=index.js.map

package/dist/node/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfigSchema,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams = PayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override configSchemas = [PayloadDivinerConfigSchema]\n\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected get payloadPairsOffset() {\n    return this.payloadPairs.at(-1)?.[1]\n  }\n\n  protected all(order: Order = 'asc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = this.payloadPairs.reverse()\n    const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInsert)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {\n    forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInsert)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: WithMeta<TOut>[]) {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n  }\n}\n"],"mappings":";;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,8BAEAC,kCAGK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,yBAAN,MAAMA,+BASHC,eAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBR;EACvC;EAEA,IAAcU,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBT;EACrC;EAEA,IAAcU,qBAAqB;AA/CrC;AAgDI,YAAO,UAAKP,aAAaQ,GAAG,EAAC,MAAtB,mBAA2B;EACpC;EAEUC,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKd,aAAaqB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAK1B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAyB2B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,kCAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAtG9B;AAuGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,CAAC,EAAEE,SAAQ,MAAE;AAC9FyB,WAAO,KAAKC,cAAc1B,QAAAA,CAAAA;EAC5B;EAEA,MAAyB2B,YAAYC,UAAiD;AACpF,UAAMjC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUkC,IAAI,YAAY,KAAK/B,iBAAiB;AAChD,WAAO,MAAM,MAAM6B,YAAAA;EACrB;;EAGA,MAAgBpB,cAAc;AAC5B,UAAM,KAAKlC,yBAAyByD,aAAa,YAAA;AAC/C,YAAMnC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIsC,cAAe,MAAMpC,UAAUqC,KAAK;QAAEpB,OAAO;QAAK9B,QAAQ,KAAKJ;MAAmB,CAAA;AACtF,aAAOqD,YAAY1B,SAAS,GAAG;AAC7BD,iBAAS,KAAKjC,aAAakC,SAAS0B,YAAY1B,UAAU,KAAK5B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKiD,cAAcK,WAAAA;AACzBA,sBAAe,MAAMpC,UAAUqC,KAAK;UAAEpB,OAAO;UAAK9B,QAAQ,KAAKJ;QAAmB,CAAA;MACpF;IACF,CAAA;EACF;EAEA,MAAcgD,cAAc1B,UAA4B;AACtD,UAAMf,QAAQ,MAAMgD,eAAeC,UAAUlC,QAAAA;AAC7C,SAAK7B,aAAagE,KAAI,GAAIlD,KAAAA;EAC5B;AACF;AAjHUf;AACR,cAVWD,wBAUKmE,iBAAgB;EAACC;;AAV5B,IAAMpE,wBAAN;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadDivinerConfigSchema","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","payloadPairsOffset","at","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","indexPayloads","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "@xyo-network/diviner-payload-generic",
+  "author": {
+    "email": "support@xyo.network",
+    "name": "XYO Development Team",
+    "url": "https://xyo.network"
+  },
+  "bugs": {
+    "email": "support@xyo.network",
+    "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js/issues"
+  },
+  "dependencies": {
+    "@xylabs/assert": "^3.1.7",
+    "@xylabs/forget": "^3.1.7",
+    "@xylabs/hex": "^3.1.7",
+    "@xylabs/object": "^3.1.7",
+    "@xyo-network/archivist-model": "~2.95.4",
+    "@xyo-network/diviner-model": "~2.95.4",
+    "@xyo-network/diviner-payload-abstract": "~2.95.4",
+    "@xyo-network/diviner-payload-model": "~2.95.4",
+    "@xyo-network/module-events": "~2.95.4",
+    "@xyo-network/payload-builder": "~2.95.4",
+    "@xyo-network/payload-model": "~2.95.4",
+    "async-mutex": "^0.5.0"
+  },
+  "devDependencies": {
+    "@xylabs/ts-scripts-yarn3": "^3.9.0",
+    "@xylabs/tsconfig": "^3.9.0",
+    "@xyo-network/account": "~2.95.4",
+    "@xyo-network/archivist-memory": "~2.95.4",
+    "@xyo-network/node-memory": "~2.95.4",
+    "@xyo-network/payload-builder": "~2.95.4",
+    "typescript": "^5.4.5"
+  },
+  "description": "Primary SDK for using XYO Protocol 2.0",
+  "types": "dist/node/index.d.ts",
+  "exports": {
+    ".": {
+      "browser": {
+        "require": {
+          "types": "./dist/browser/index.d.cts",
+          "default": "./dist/browser/index.cjs"
+        },
+        "import": {
+          "types": "./dist/browser/index.d.mts",
+          "default": "./dist/browser/index.js"
+        }
+      },
+      "node": {
+        "require": {
+          "types": "./dist/node/index.d.cts",
+          "default": "./dist/node/index.cjs"
+        },
+        "import": {
+          "types": "./dist/node/index.d.mts",
+          "default": "./dist/node/index.js"
+        }
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "main": "dist/node/index.cjs",
+  "module": "dist/node/index.js",
+  "homepage": "https://xyo.network",
+  "license": "LGPL-3.0-only",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
+  },
+  "sideEffects": false,
+  "version": "2.95.4",
+  "type": "module"
+}

package/src/Diviner.ts

@@ -0,0 +1,145 @@
+import { assertEx } from '@xylabs/assert'
+import { forget } from '@xylabs/forget'
+import { Hash } from '@xylabs/hex'
+import { EmptyObject } from '@xylabs/object'
+import { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'
+import { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'
+import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'
+import {
+  isPayloadDivinerQueryPayload,
+  Order,
+  PayloadDivinerConfigSchema,
+  PayloadDivinerParams,
+  PayloadDivinerQueryPayload,
+} from '@xyo-network/diviner-payload-model'
+import { EventListener } from '@xyo-network/module-events'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import { Payload, WithMeta } from '@xyo-network/payload-model'
+import { Mutex } from 'async-mutex'
+
+const DEFAULT_INDEX_BATCH_SIZE = 100 as const
+const DEFAULT_MAX_INDEX_SIZE = 8000 as const
+
+export class GenericPayloadDiviner<
+  TParams extends PayloadDivinerParams = PayloadDivinerParams,
+  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,
+  TOut extends Payload = Payload,
+  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<
+    DivinerInstance<TParams, TIn, TOut>,
+    TIn,
+    TOut
+  >,
+> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {
+  static override configSchemas = [PayloadDivinerConfigSchema]
+
+  protected payloadPairs: [WithMeta<TOut>, Hash][] = []
+
+  private _archivistInstance?: ArchivistInstance
+  private _updatePayloadPairsMutex = new Mutex()
+
+  protected get indexBatchSize() {
+    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE
+  }
+
+  protected get maxIndexSize() {
+    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE
+  }
+
+  protected get payloadPairsOffset() {
+    return this.payloadPairs.at(-1)?.[1]
+  }
+
+  protected all(order: Order = 'asc', offset?: Hash) {
+    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)
+  }
+
+  protected allAsc(offset?: Hash) {
+    const pairs = this.payloadPairs
+    const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0
+    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)
+  }
+
+  protected allDesc(offset?: Hash) {
+    const pairs = this.payloadPairs.reverse()
+    const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0
+    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)
+  }
+
+  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>
+  protected override async archivistInstance(required: true): Promise<ArchivistInstance>
+  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {
+    if (!this._archivistInstance) {
+      const archivist = await super.archivistInstance()
+      if (required && !archivist) {
+        throw new Error('Failed to find archivist')
+      }
+      archivist?.on('inserted', this.onArchivistInsert)
+      this._archivistInstance = archivist
+    }
+    return this._archivistInstance
+  }
+
+  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {
+    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []
+    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')
+    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<
+      PayloadDivinerQueryPayload<EmptyObject, Hash>
+    >
+
+    await this.updateIndex()
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter
+    let all: WithMeta<TOut>[] = this.all(order, offset)
+    if (all) {
+      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))
+      if (Object.keys(props).length > 0) {
+        const additionalFilterCriteria = Object.entries(props)
+        for (const [prop, filter] of additionalFilterCriteria) {
+          const property = prop as keyof TOut
+          all =
+            Array.isArray(filter) ?
+              all.filter((payload) =>
+                filter.every((value) => {
+                  const prop = payload?.[property]
+                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?
+                  return Array.isArray(prop) && prop.includes?.(value)
+                }),
+              )
+            : all.filter((payload) => payload?.[property] === filter)
+        }
+      }
+      return limit ? all.slice(0, limit) : all
+    } else {
+      throw new Error('Archivist does not support "all"')
+    }
+  }
+
+  protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {
+    forget(this.indexPayloads(payloads as WithMeta<TOut>[]))
+  }
+
+  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {
+    const archivist = await this.archivistInstance(true)
+    archivist.off('inserted', this.onArchivistInsert)
+    return await super.stopHandler()
+  }
+
+  //index any new payloads
+  protected async updateIndex() {
+    await this._updatePayloadPairsMutex.runExclusive(async () => {
+      const archivist = await this.archivistInstance(true)
+      let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]
+      while (newPayloads.length > 0) {
+        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')
+        await this.indexPayloads(newPayloads)
+        newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]
+      }
+    })
+  }
+
+  private async indexPayloads(payloads: WithMeta<TOut>[]) {
+    const pairs = await PayloadBuilder.hashPairs(payloads)
+    this.payloadPairs.push(...pairs)
+  }
+}

package/src/index.ts

@@ -0,0 +1 @@
+export * from './Diviner'

package/typedoc.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://typedoc.org/schema.json",
+  "entryPoints": ["src/index.ts"],
+  "tsconfig": "./tsconfig.typedoc.json"
+}