@xyo-network/diviner-payload-generic 2.96.1 → 2.97.0

package/dist/browser/Diviner.d.cts

@@ -4,11 +4,12 @@ import { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-mo
 import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract';
 import { Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model';
 import { EventListener } from '@xyo-network/module-events';
-import { Payload, WithMeta } from '@xyo-network/payload-model';
+import { Payload, Schema, WithMeta } from '@xyo-network/payload-model';
 export declare const GenericPayloadDivinerConfigSchema: "network.xyo.diviner.payload.generic.config";
 export type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema;
 export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams = PayloadDivinerParams, TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut>> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {
-    static configSchemas: "network.xyo.diviner.payload.config"[];
+    static configSchemas: Schema[];
+    static defaultConfigSchema: Schema;
     protected payloadPairs: [WithMeta<TOut>, Hash][];
     private _archivistInstance?;
     private _indexOffset?;

package/dist/browser/Diviner.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,EAAE,MAAM,EAAE,CAA8D;IACrG,OAAgB,mBAAmB,EAAE,MAAM,CAAoC;IAE/E,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}

package/dist/browser/Diviner.d.mts

@@ -4,11 +4,12 @@ import { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-mo
 import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract';
 import { Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model';
 import { EventListener } from '@xyo-network/module-events';
-import { Payload, WithMeta } from '@xyo-network/payload-model';
+import { Payload, Schema, WithMeta } from '@xyo-network/payload-model';
 export declare const GenericPayloadDivinerConfigSchema: "network.xyo.diviner.payload.generic.config";
 export type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema;
 export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams = PayloadDivinerParams, TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut>> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {
-    static configSchemas: "network.xyo.diviner.payload.config"[];
+    static configSchemas: Schema[];
+    static defaultConfigSchema: Schema;
     protected payloadPairs: [WithMeta<TOut>, Hash][];
     private _archivistInstance?;
     private _indexOffset?;

package/dist/browser/Diviner.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,EAAE,MAAM,EAAE,CAA8D;IACrG,OAAgB,mBAAmB,EAAE,MAAM,CAAoC;IAE/E,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}

package/dist/browser/Diviner.d.ts

@@ -4,11 +4,12 @@ import { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-mo
 import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract';
 import { Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model';
 import { EventListener } from '@xyo-network/module-events';
-import { Payload, WithMeta } from '@xyo-network/payload-model';
+import { Payload, Schema, WithMeta } from '@xyo-network/payload-model';
 export declare const GenericPayloadDivinerConfigSchema: "network.xyo.diviner.payload.generic.config";
 export type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema;
 export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams = PayloadDivinerParams, TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut>> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {
-    static configSchemas: "network.xyo.diviner.payload.config"[];
+    static configSchemas: Schema[];
+    static defaultConfigSchema: Schema;
     protected payloadPairs: [WithMeta<TOut>, Hash][];
     private _archivistInstance?;
     private _indexOffset?;

package/dist/browser/Diviner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,EAAE,MAAM,EAAE,CAA8D;IACrG,OAAgB,mBAAmB,EAAE,MAAM,CAAoC;IAE/E,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}

package/dist/browser/index.cjs

@@ -41,8 +41,10 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
     __name(this, "GenericPayloadDiviner");
   }
   static configSchemas = [
-    import_diviner_payload_model.PayloadDivinerConfigSchema
+    ...super.configSchemas,
+    GenericPayloadDivinerConfigSchema
   ];
+  static defaultConfigSchema = GenericPayloadDivinerConfigSchema;
   payloadPairs = [];
   _archivistInstance;
   _indexOffset;
@@ -119,8 +121,11 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
         offset: this._indexOffset
       });
       while (newPayloads.length > 0) {
-        console.log("newPayloads", newPayloads);
+        const prevOffset = this._indexOffset;
         this._indexOffset = await import_payload_builder.PayloadBuilder.hash((0, import_assert.assertEx)(newPayloads.at(-1)));
+        if (this._indexOffset === prevOffset) {
+          this.logger.warn("next offset not found", prevOffset);
+        }
         (0, import_assert.assertEx)(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
         await this.indexPayloads(newPayloads);
         newPayloads = await archivist.next({

package/dist/browser/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfigSchema,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams = PayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override configSchemas = [PayloadDivinerConfigSchema]\n\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'asc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = this.payloadPairs.reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInsert)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInsert)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        console.log('newPayloads', newPayloads)\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,+CAAAA;EAlCV,OAkCUA;;;EACR,OAAgBC,gBAAgB;IAACC;;EAEvBC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B2B,gBAAQC,IAAI,eAAeH,WAAAA;AAC3B,aAAKxD,eAAe,MAAM4D,sCAAe9C,SAAKgB,wBAAS0B,YAAYK,GAAG,EAAC,CAAA,CAAA;AACvE/B,oCAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKyD,cAAcN,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc8D,cAAcpC,UAA2C;AACrE,UAAMf,QAAQ,MAAMiD,sCAAeG,UAAUrC,QAAAA;AAC7C,SAAK5B,aAAakE,KAAI,GAAIrD,KAAAA;AAC1B,eAAOmB,wBAASnB,MAAMkD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","PayloadDivinerConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","console","log","PayloadBuilder","at","indexPayloads","hashPairs","push"]}
+{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams = PayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'asc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = this.payloadPairs.reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInsert)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInsert)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAAsG;AAEtG,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,+CAAAA;EA5BV,OA4BUA;;;EACR,OAAgBC,gBAA0B;OAAI,MAAMA;IAAeH;;EACnE,OAAgBI,sBAA8BJ;EAEpCK,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,cAAM2B,aAAa,KAAK1D;AACxB,aAAKA,eAAe,MAAM2D,sCAAe7C,SAAKgB,wBAAS0B,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAK5D,iBAAiB0D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA5B,oCAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK0D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc+D,cAAcrC,UAA2C;AACrE,UAAMf,QAAQ,MAAMgD,sCAAeK,UAAUtC,QAAAA;AAC7C,SAAK5B,aAAamE,KAAI,GAAItD,KAAAA;AAC1B,eAAOmB,wBAASnB,MAAMiD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push"]}

package/dist/browser/index.js

@@ -5,7 +5,7 @@ var __name = (target, value) => __defProp(target, "name", { value, configurable:
 import { assertEx } from "@xylabs/assert";
 import { forget } from "@xylabs/forget";
 import { PayloadDiviner } from "@xyo-network/diviner-payload-abstract";
-import { isPayloadDivinerQueryPayload, PayloadDivinerConfigSchema } from "@xyo-network/diviner-payload-model";
+import { isPayloadDivinerQueryPayload } from "@xyo-network/diviner-payload-model";
 import { PayloadBuilder } from "@xyo-network/payload-builder";
 import { Mutex } from "async-mutex";
 var DEFAULT_INDEX_BATCH_SIZE = 100;
@@ -16,8 +16,10 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
     __name(this, "GenericPayloadDiviner");
   }
   static configSchemas = [
-    PayloadDivinerConfigSchema
+    ...super.configSchemas,
+    GenericPayloadDivinerConfigSchema
   ];
+  static defaultConfigSchema = GenericPayloadDivinerConfigSchema;
   payloadPairs = [];
   _archivistInstance;
   _indexOffset;
@@ -94,8 +96,11 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
         offset: this._indexOffset
       });
       while (newPayloads.length > 0) {
-        console.log("newPayloads", newPayloads);
+        const prevOffset = this._indexOffset;
         this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)));
+        if (this._indexOffset === prevOffset) {
+          this.logger.warn("next offset not found", prevOffset);
+        }
         assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
         await this.indexPayloads(newPayloads);
         newPayloads = await archivist.next({

package/dist/browser/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfigSchema,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams = PayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override configSchemas = [PayloadDivinerConfigSchema]\n\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'asc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = this.payloadPairs.reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInsert)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInsert)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        console.log('newPayloads', newPayloads)\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,8BAEAC,kCAGK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,eAAAA;EAlCV,OAkCUA;;;EACR,OAAgBC,gBAAgB;IAACC;;EAEvBC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B2B,gBAAQC,IAAI,eAAeH,WAAAA;AAC3B,aAAKxD,eAAe,MAAM4D,eAAe9C,KAAKgB,SAAS0B,YAAYK,GAAG,EAAC,CAAA,CAAA;AACvE/B,iBAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKyD,cAAcN,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc8D,cAAcpC,UAA2C;AACrE,UAAMf,QAAQ,MAAMiD,eAAeG,UAAUrC,QAAAA;AAC7C,SAAK5B,aAAakE,KAAI,GAAIrD,KAAAA;AAC1B,WAAOmB,SAASnB,MAAMkD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadDivinerConfigSchema","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","PayloadDivinerConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","console","log","PayloadBuilder","at","indexPayloads","hashPairs","push"]}
+{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams = PayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'asc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = this.payloadPairs.reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInsert)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInsert)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SAASC,oCAA6F;AAEtG,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,eAAAA;EA5BV,OA4BUA;;;EACR,OAAgBC,gBAA0B;OAAI,MAAMA;IAAeH;;EACnE,OAAgBI,sBAA8BJ;EAEpCK,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,cAAM2B,aAAa,KAAK1D;AACxB,aAAKA,eAAe,MAAM2D,eAAe7C,KAAKgB,SAAS0B,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAK5D,iBAAiB0D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA5B,iBAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK0D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc+D,cAAcrC,UAA2C;AACrE,UAAMf,QAAQ,MAAMgD,eAAeK,UAAUtC,QAAAA;AAC7C,SAAK5B,aAAamE,KAAI,GAAItD,KAAAA;AAC1B,WAAOmB,SAASnB,MAAMiD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push"]}

package/dist/node/Diviner.d.cts

@@ -4,11 +4,12 @@ import { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-mo
 import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract';
 import { Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model';
 import { EventListener } from '@xyo-network/module-events';
-import { Payload, WithMeta } from '@xyo-network/payload-model';
+import { Payload, Schema, WithMeta } from '@xyo-network/payload-model';
 export declare const GenericPayloadDivinerConfigSchema: "network.xyo.diviner.payload.generic.config";
 export type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema;
 export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams = PayloadDivinerParams, TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut>> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {
-    static configSchemas: "network.xyo.diviner.payload.config"[];
+    static configSchemas: Schema[];
+    static defaultConfigSchema: Schema;
     protected payloadPairs: [WithMeta<TOut>, Hash][];
     private _archivistInstance?;
     private _indexOffset?;

package/dist/node/Diviner.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,EAAE,MAAM,EAAE,CAA8D;IACrG,OAAgB,mBAAmB,EAAE,MAAM,CAAoC;IAE/E,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}

package/dist/node/Diviner.d.mts

@@ -4,11 +4,12 @@ import { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-mo
 import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract';
 import { Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model';
 import { EventListener } from '@xyo-network/module-events';
-import { Payload, WithMeta } from '@xyo-network/payload-model';
+import { Payload, Schema, WithMeta } from '@xyo-network/payload-model';
 export declare const GenericPayloadDivinerConfigSchema: "network.xyo.diviner.payload.generic.config";
 export type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema;
 export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams = PayloadDivinerParams, TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut>> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {
-    static configSchemas: "network.xyo.diviner.payload.config"[];
+    static configSchemas: Schema[];
+    static defaultConfigSchema: Schema;
     protected payloadPairs: [WithMeta<TOut>, Hash][];
     private _archivistInstance?;
     private _indexOffset?;

package/dist/node/Diviner.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,EAAE,MAAM,EAAE,CAA8D;IACrG,OAAgB,mBAAmB,EAAE,MAAM,CAAoC;IAE/E,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}

package/dist/node/Diviner.d.ts

@@ -4,11 +4,12 @@ import { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-mo
 import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract';
 import { Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model';
 import { EventListener } from '@xyo-network/module-events';
-import { Payload, WithMeta } from '@xyo-network/payload-model';
+import { Payload, Schema, WithMeta } from '@xyo-network/payload-model';
 export declare const GenericPayloadDivinerConfigSchema: "network.xyo.diviner.payload.generic.config";
 export type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema;
 export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams = PayloadDivinerParams, TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut>> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {
-    static configSchemas: "network.xyo.diviner.payload.config"[];
+    static configSchemas: Schema[];
+    static defaultConfigSchema: Schema;
     protected payloadPairs: [WithMeta<TOut>, Hash][];
     private _archivistInstance?;
     private _indexOffset?;

package/dist/node/Diviner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,EAAE,MAAM,EAAE,CAA8D;IACrG,OAAgB,mBAAmB,EAAE,MAAM,CAAoC;IAE/E,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}

package/dist/node/index.cjs

@@ -2,7 +2,9 @@
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __reflectGet = Reflect.get;
 var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
 var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
 var __export = (target, all) => {
@@ -22,6 +24,7 @@ var __publicField = (obj, key, value) => {
   __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
   return value;
 };
+var __superGet = (cls, obj, key) => __reflectGet(__getProtoOf(cls), key, obj);
 
 // src/index.ts
 var src_exports = {};
@@ -119,8 +122,11 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
         offset: this._indexOffset
       });
       while (newPayloads.length > 0) {
-        console.log("newPayloads", newPayloads);
+        const prevOffset = this._indexOffset;
         this._indexOffset = await import_payload_builder.PayloadBuilder.hash((0, import_assert.assertEx)(newPayloads.at(-1)));
+        if (this._indexOffset === prevOffset) {
+          this.logger.warn("next offset not found", prevOffset);
+        }
         (0, import_assert.assertEx)(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
         await this.indexPayloads(newPayloads);
         newPayloads = await archivist.next({
@@ -138,8 +144,10 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
 };
 __name(_GenericPayloadDiviner, "GenericPayloadDiviner");
 __publicField(_GenericPayloadDiviner, "configSchemas", [
-  import_diviner_payload_model.PayloadDivinerConfigSchema
+  ...__superGet(_GenericPayloadDiviner, _GenericPayloadDiviner, "configSchemas"),
+  GenericPayloadDivinerConfigSchema
 ]);
+__publicField(_GenericPayloadDiviner, "defaultConfigSchema", GenericPayloadDivinerConfigSchema);
 var GenericPayloadDiviner = _GenericPayloadDiviner;
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/node/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfigSchema,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams = PayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override configSchemas = [PayloadDivinerConfigSchema]\n\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'asc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = this.payloadPairs.reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInsert)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInsert)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        console.log('newPayloads', newPayloads)\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,yBAAN,MAAMA,+BASHC,+CAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEUY,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,+DAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAtG9B;AAuGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B2B,gBAAQC,IAAI,eAAeH,WAAAA;AAC3B,aAAKxD,eAAe,MAAM4D,sCAAe9C,SAAKgB,wBAAS0B,YAAYK,GAAG,EAAC,CAAA,CAAA;AACvE/B,oCAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKyD,cAAcN,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc8D,cAAcpC,UAA2C;AACrE,UAAMf,QAAQ,MAAMiD,sCAAeG,UAAUrC,QAAAA;AAC7C,SAAK5B,aAAakE,KAAI,GAAIrD,KAAAA;AAC1B,eAAOmB,wBAASnB,MAAMkD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAlHUhE;AACR,cAVWD,wBAUKqE,iBAAgB;EAACC;;AAV5B,IAAMtE,wBAAN;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","console","log","PayloadBuilder","at","indexPayloads","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}
+{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams = PayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'asc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = this.payloadPairs.reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInsert)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInsert)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAAsG;AAEtG,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,yBAAN,MAAMA,+BASHC,+CAAAA;EAIEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEUY,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,+DAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAjG9B;AAkGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,cAAM2B,aAAa,KAAK1D;AACxB,aAAKA,eAAe,MAAM2D,sCAAe7C,SAAKgB,wBAAS0B,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAK5D,iBAAiB0D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA5B,oCAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK0D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc+D,cAAcrC,UAA2C;AACrE,UAAMf,QAAQ,MAAMgD,sCAAeK,UAAUtC,QAAAA;AAC7C,SAAK5B,aAAamE,KAAI,GAAItD,KAAAA;AAC1B,eAAOmB,wBAASnB,MAAMiD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAtHU/D;AACR,cAVWD,wBAUKsE,iBAA0B;KAAI,2DAAMA;EAAevE;;AACnE,cAXWC,wBAWKuE,uBAA8BxE;AAXzC,IAAMC,wBAAN;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","configSchemas","defaultConfigSchema"]}

package/dist/node/index.js

@@ -1,16 +1,19 @@
 var __defProp = Object.defineProperty;
+var __getProtoOf = Object.getPrototypeOf;
+var __reflectGet = Reflect.get;
 var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
 var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
 var __publicField = (obj, key, value) => {
   __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
   return value;
 };
+var __superGet = (cls, obj, key) => __reflectGet(__getProtoOf(cls), key, obj);
 
 // src/Diviner.ts
 import { assertEx } from "@xylabs/assert";
 import { forget } from "@xylabs/forget";
 import { PayloadDiviner } from "@xyo-network/diviner-payload-abstract";
-import { isPayloadDivinerQueryPayload, PayloadDivinerConfigSchema } from "@xyo-network/diviner-payload-model";
+import { isPayloadDivinerQueryPayload } from "@xyo-network/diviner-payload-model";
 import { PayloadBuilder } from "@xyo-network/payload-builder";
 import { Mutex } from "async-mutex";
 var DEFAULT_INDEX_BATCH_SIZE = 100;
@@ -94,8 +97,11 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
         offset: this._indexOffset
       });
       while (newPayloads.length > 0) {
-        console.log("newPayloads", newPayloads);
+        const prevOffset = this._indexOffset;
         this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)));
+        if (this._indexOffset === prevOffset) {
+          this.logger.warn("next offset not found", prevOffset);
+        }
         assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
         await this.indexPayloads(newPayloads);
         newPayloads = await archivist.next({
@@ -113,8 +119,10 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
 };
 __name(_GenericPayloadDiviner, "GenericPayloadDiviner");
 __publicField(_GenericPayloadDiviner, "configSchemas", [
-  PayloadDivinerConfigSchema
+  ...__superGet(_GenericPayloadDiviner, _GenericPayloadDiviner, "configSchemas"),
+  GenericPayloadDivinerConfigSchema
 ]);
+__publicField(_GenericPayloadDiviner, "defaultConfigSchema", GenericPayloadDivinerConfigSchema);
 var GenericPayloadDiviner = _GenericPayloadDiviner;
 export {
   GenericPayloadDiviner,

package/dist/node/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfigSchema,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams = PayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override configSchemas = [PayloadDivinerConfigSchema]\n\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'asc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = this.payloadPairs.reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInsert)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInsert)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        console.log('newPayloads', newPayloads)\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,8BAEAC,kCAGK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,yBAAN,MAAMA,+BASHC,eAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEUY,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,kCAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAtG9B;AAuGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B2B,gBAAQC,IAAI,eAAeH,WAAAA;AAC3B,aAAKxD,eAAe,MAAM4D,eAAe9C,KAAKgB,SAAS0B,YAAYK,GAAG,EAAC,CAAA,CAAA;AACvE/B,iBAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKyD,cAAcN,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc8D,cAAcpC,UAA2C;AACrE,UAAMf,QAAQ,MAAMiD,eAAeG,UAAUrC,QAAAA;AAC7C,SAAK5B,aAAakE,KAAI,GAAIrD,KAAAA;AAC1B,WAAOmB,SAASnB,MAAMkD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAlHUhE;AACR,cAVWD,wBAUKqE,iBAAgB;EAACC;;AAV5B,IAAMtE,wBAAN;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadDivinerConfigSchema","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","console","log","PayloadBuilder","at","indexPayloads","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}
+{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams = PayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'asc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = this.payloadPairs.reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInsert)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInsert)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SAASC,oCAA6F;AAEtG,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,yBAAN,MAAMA,+BASHC,eAAAA;EAIEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEUY,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,kCAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAjG9B;AAkGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,cAAM2B,aAAa,KAAK1D;AACxB,aAAKA,eAAe,MAAM2D,eAAe7C,KAAKgB,SAAS0B,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAK5D,iBAAiB0D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA5B,iBAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK0D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc+D,cAAcrC,UAA2C;AACrE,UAAMf,QAAQ,MAAMgD,eAAeK,UAAUtC,QAAAA;AAC7C,SAAK5B,aAAamE,KAAI,GAAItD,KAAAA;AAC1B,WAAOmB,SAASnB,MAAMiD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAtHU/D;AACR,cAVWD,wBAUKsE,iBAA0B;KAAI,2DAAMA;EAAevE;;AACnE,cAXWC,wBAWKuE,uBAA8BxE;AAXzC,IAAMC,wBAAN;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","configSchemas","defaultConfigSchema"]}

package/package.json

@@ -14,22 +14,22 @@
     "@xylabs/forget": "^3.1.8",
     "@xylabs/hex": "^3.1.8",
     "@xylabs/object": "^3.1.8",
-    "@xyo-network/archivist-model": "~2.96.1",
-    "@xyo-network/diviner-model": "~2.96.1",
-    "@xyo-network/diviner-payload-abstract": "~2.96.1",
-    "@xyo-network/diviner-payload-model": "~2.96.1",
-    "@xyo-network/module-events": "~2.96.1",
-    "@xyo-network/payload-builder": "~2.96.1",
-    "@xyo-network/payload-model": "~2.96.1",
+    "@xyo-network/archivist-model": "~2.97.0",
+    "@xyo-network/diviner-model": "~2.97.0",
+    "@xyo-network/diviner-payload-abstract": "~2.97.0",
+    "@xyo-network/diviner-payload-model": "~2.97.0",
+    "@xyo-network/module-events": "~2.97.0",
+    "@xyo-network/payload-builder": "~2.97.0",
+    "@xyo-network/payload-model": "~2.97.0",
     "async-mutex": "^0.5.0"
   },
   "devDependencies": {
     "@xylabs/ts-scripts-yarn3": "^3.9.1",
     "@xylabs/tsconfig": "^3.9.1",
-    "@xyo-network/account": "~2.96.1",
-    "@xyo-network/archivist-memory": "~2.96.1",
-    "@xyo-network/node-memory": "~2.96.1",
-    "@xyo-network/payload-builder": "~2.96.1",
+    "@xyo-network/account": "~2.97.0",
+    "@xyo-network/archivist-memory": "~2.97.0",
+    "@xyo-network/node-memory": "~2.97.0",
+    "@xyo-network/payload-builder": "~2.97.0",
     "typescript": "^5.4.5"
   },
   "description": "Primary SDK for using XYO Protocol 2.0",
@@ -71,6 +71,6 @@
     "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
   },
   "sideEffects": false,
-  "version": "2.96.1",
+  "version": "2.97.0",
   "type": "module"
 }

package/src/Diviner.ts

@@ -5,16 +5,10 @@ import { EmptyObject } from '@xylabs/object'
 import { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'
 import { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'
 import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'
-import {
-  isPayloadDivinerQueryPayload,
-  Order,
-  PayloadDivinerConfigSchema,
-  PayloadDivinerParams,
-  PayloadDivinerQueryPayload,
-} from '@xyo-network/diviner-payload-model'
+import { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
 import { EventListener } from '@xyo-network/module-events'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
-import { Payload, WithMeta } from '@xyo-network/payload-model'
+import { Payload, Schema, WithMeta } from '@xyo-network/payload-model'
 import { Mutex } from 'async-mutex'
 
 const DEFAULT_INDEX_BATCH_SIZE = 100 as const
@@ -33,7 +27,8 @@ export class GenericPayloadDiviner<
     TOut
   >,
 > extends PayloadDiviner<TParams, TIn, TOut, TEventData> {
-  static override configSchemas = [PayloadDivinerConfigSchema]
+  static override configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]
+  static override defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema
 
   protected payloadPairs: [WithMeta<TOut>, Hash][] = []
 
@@ -132,8 +127,11 @@ export class GenericPayloadDiviner<
       const archivist = await this.archivistInstance(true)
       let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]
       while (newPayloads.length > 0) {
-        console.log('newPayloads', newPayloads)
+        const prevOffset = this._indexOffset
         this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))
+        if (this._indexOffset === prevOffset) {
+          this.logger.warn('next offset not found', prevOffset)
+        }
         assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')
         await this.indexPayloads(newPayloads)
         newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]