@xyo-network/diviner-payload-generic 2.100.0 → 2.100.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/browser/Diviner.d.cts +4 -1
  2. package/dist/browser/Diviner.d.cts.map +1 -1
  3. package/dist/browser/Diviner.d.mts +4 -1
  4. package/dist/browser/Diviner.d.mts.map +1 -1
  5. package/dist/browser/Diviner.d.ts +4 -1
  6. package/dist/browser/Diviner.d.ts.map +1 -1
  7. package/dist/browser/index.cjs +26 -3
  8. package/dist/browser/index.cjs.map +1 -1
  9. package/dist/browser/index.js +26 -3
  10. package/dist/browser/index.js.map +1 -1
  11. package/dist/neutral/Diviner.d.cts +4 -1
  12. package/dist/neutral/Diviner.d.cts.map +1 -1
  13. package/dist/neutral/Diviner.d.mts +4 -1
  14. package/dist/neutral/Diviner.d.mts.map +1 -1
  15. package/dist/neutral/Diviner.d.ts +4 -1
  16. package/dist/neutral/Diviner.d.ts.map +1 -1
  17. package/dist/neutral/index.cjs +26 -3
  18. package/dist/neutral/index.cjs.map +1 -1
  19. package/dist/neutral/index.js +26 -3
  20. package/dist/neutral/index.js.map +1 -1
  21. package/dist/node/Diviner.d.cts +4 -1
  22. package/dist/node/Diviner.d.cts.map +1 -1
  23. package/dist/node/Diviner.d.mts +4 -1
  24. package/dist/node/Diviner.d.mts.map +1 -1
  25. package/dist/node/Diviner.d.ts +4 -1
  26. package/dist/node/Diviner.d.ts.map +1 -1
  27. package/dist/node/index.cjs +26 -3
  28. package/dist/node/index.cjs.map +1 -1
  29. package/dist/node/index.js +26 -3
  30. package/dist/node/index.js.map +1 -1
  31. package/package.json +12 -12
  32. package/src/Diviner.ts +33 -4
package/dist/browser/Diviner.d.cts CHANGED
@@ -21,8 +21,11 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
21
21
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
22
22
  protected archivistInstance(): Promise<ArchivistInstance | undefined>;
23
23
  protected archivistInstance(required: true): Promise<ArchivistInstance>;
24
+ protected clearIndex(): Promise<void>;
24
25
  protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
25
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']>;
26
+ protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']>;
27
+ protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']>;
28
+ protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']>;
26
29
  protected stopHandler(_timeout?: number | undefined): Promise<boolean>;
27
30
  protected updateIndex(): Promise<void>;
28
31
  private indexPayloads;
package/dist/browser/Diviner.d.cts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAetE,UAAU;cAOD,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAGD,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAED,SAAS,CAAC,mBAAmB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAEjF;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cASrE,WAAW;YAiBb,aAAa;CAK5B"}
package/dist/browser/Diviner.d.mts CHANGED
@@ -21,8 +21,11 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
21
21
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
22
22
  protected archivistInstance(): Promise<ArchivistInstance | undefined>;
23
23
  protected archivistInstance(required: true): Promise<ArchivistInstance>;
24
+ protected clearIndex(): Promise<void>;
24
25
  protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
25
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']>;
26
+ protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']>;
27
+ protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']>;
28
+ protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']>;
26
29
  protected stopHandler(_timeout?: number | undefined): Promise<boolean>;
27
30
  protected updateIndex(): Promise<void>;
28
31
  private indexPayloads;
package/dist/browser/Diviner.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAetE,UAAU;cAOD,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAGD,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAED,SAAS,CAAC,mBAAmB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAEjF;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cASrE,WAAW;YAiBb,aAAa;CAK5B"}
package/dist/browser/Diviner.d.ts CHANGED
@@ -21,8 +21,11 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
21
21
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
22
22
  protected archivistInstance(): Promise<ArchivistInstance | undefined>;
23
23
  protected archivistInstance(required: true): Promise<ArchivistInstance>;
24
+ protected clearIndex(): Promise<void>;
24
25
  protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
25
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']>;
26
+ protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']>;
27
+ protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']>;
28
+ protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']>;
26
29
  protected stopHandler(_timeout?: number | undefined): Promise<boolean>;
27
30
  protected updateIndex(): Promise<void>;
28
31
  private indexPayloads;
package/dist/browser/Diviner.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAetE,UAAU;cAOD,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAGD,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAED,SAAS,CAAC,mBAAmB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAEjF;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cASrE,WAAW;YAiBb,aAAa;CAK5B"}
package/dist/browser/index.cjs CHANGED
@@ -76,11 +76,19 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
76
76
  if (required && !archivist) {
77
77
  throw new Error("Failed to find archivist");
78
78
  }
79
- archivist?.on("inserted", this.onArchivistInsert);
79
+ archivist?.on("inserted", this.onArchivistInserted);
80
+ archivist?.on("cleared", this.onArchivistCleared);
81
+ archivist?.on("deleted", this.onArchivistDeleted);
80
82
  this._archivistInstance = archivist;
81
83
  }
82
84
  return this._archivistInstance;
83
85
  }
86
+ async clearIndex() {
87
+ await this._updatePayloadPairsMutex.runExclusive(() => {
88
+ this._indexOffset = void 0;
89
+ this.payloadPairs = [];
90
+ });
91
+ }
84
92
  async divineHandler(payloads) {
85
93
  const filters = payloads?.filter(import_diviner_payload_model.isPayloadDivinerQueryPayload) ?? [];
86
94
  (0, import_assert.assertEx)(filters.length < 2, () => "Multiple PayloadDivinerQuery payloads may not be specified");
@@ -106,12 +114,27 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
106
114
  throw new Error('Archivist does not support "all"');
107
115
  }
108
116
  }
109
- onArchivistInsert = () => {
117
+ onArchivistCleared = () => {
118
+ (0, import_forget.forget)((async () => {
119
+ await this.clearIndex();
120
+ await this.updateIndex();
121
+ })());
122
+ };
123
+ // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
124
+ onArchivistDeleted = () => {
125
+ (0, import_forget.forget)((async () => {
126
+ await this.clearIndex();
127
+ await this.updateIndex();
128
+ })());
129
+ };
130
+ onArchivistInserted = () => {
110
131
  (0, import_forget.forget)(this.updateIndex());
111
132
  };
112
133
  async stopHandler(_timeout) {
113
134
  const archivist = await this.archivistInstance(true);
114
- archivist.off("inserted", this.onArchivistInsert);
135
+ archivist.off("inserted", this.onArchivistInserted);
136
+ archivist.off("deleted", this.onArchivistDeleted);
137
+ archivist.off("cleared", this.onArchivistCleared);
115
138
  return await super.stopHandler();
116
139
  }
117
140
  //index any new payloads
package/dist/browser/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAAsG;AAEtG,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,+CAAAA;EA5BV,OA4BUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,cAAM2B,aAAa,KAAK1D;AACxB,aAAKA,eAAe,MAAM2D,sCAAe7C,SAAKgB,wBAAS0B,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAK5D,iBAAiB0D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA5B,oCAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK0D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc+D,cAAcrC,UAA2C;AACrE,UAAMf,QAAQ,MAAMgD,sCAAeK,UAAUtC,QAAAA;AAC7C,SAAK5B,aAAamE,KAAI,GAAItD,KAAAA;AAC1B,eAAOmB,wBAASnB,MAAMiD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push"]}
1
+ {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInserted)\n archivist?.on('cleared', this.onArchivistCleared)\n archivist?.on('deleted', this.onArchivistDeleted)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected async clearIndex() {\n await this._updatePayloadPairsMutex.runExclusive(() => {\n this._indexOffset = undefined\n this.payloadPairs = []\n })\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInserted)\n archivist.off('deleted', this.onArchivistDeleted)\n archivist.off('cleared', this.onArchivistCleared)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAAsG;AAEtG,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,+CAAAA;EA5BV,OA4BUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK3B,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAgB4B,aAAa;AAC3B,UAAM,KAAK1B,yBAAyB2B,aAAa,MAAA;AAC/C,WAAK5B,eAAe6B;AACpB,WAAK/B,eAAe,CAAA;IACtB,CAAA;EACF;EAEA,MAAyBgC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKrC,yBAAyB2B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAO4D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK9D;AACxB,aAAKA,eAAe,MAAM+D,sCAAejD,SAAKqB,wBAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKhE,iBAAiB8D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,oCAAS,KAAKrC,aAAasC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcmE,cAAcpC,UAA2C;AACrE,UAAMpB,QAAQ,MAAMoD,sCAAeK,UAAUrC,QAAAA;AAC7C,SAAKjC,aAAauE,KAAI,GAAI1D,KAAAA;AAC1B,eAAOwB,wBAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push"]}
package/dist/browser/index.js CHANGED
@@ -51,11 +51,19 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
51
51
  if (required && !archivist) {
52
52
  throw new Error("Failed to find archivist");
53
53
  }
54
- archivist?.on("inserted", this.onArchivistInsert);
54
+ archivist?.on("inserted", this.onArchivistInserted);
55
+ archivist?.on("cleared", this.onArchivistCleared);
56
+ archivist?.on("deleted", this.onArchivistDeleted);
55
57
  this._archivistInstance = archivist;
56
58
  }
57
59
  return this._archivistInstance;
58
60
  }
61
+ async clearIndex() {
62
+ await this._updatePayloadPairsMutex.runExclusive(() => {
63
+ this._indexOffset = void 0;
64
+ this.payloadPairs = [];
65
+ });
66
+ }
59
67
  async divineHandler(payloads) {
60
68
  const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? [];
61
69
  assertEx(filters.length < 2, () => "Multiple PayloadDivinerQuery payloads may not be specified");
@@ -81,12 +89,27 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
81
89
  throw new Error('Archivist does not support "all"');
82
90
  }
83
91
  }
84
- onArchivistInsert = () => {
92
+ onArchivistCleared = () => {
93
+ forget((async () => {
94
+ await this.clearIndex();
95
+ await this.updateIndex();
96
+ })());
97
+ };
98
+ // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
99
+ onArchivistDeleted = () => {
100
+ forget((async () => {
101
+ await this.clearIndex();
102
+ await this.updateIndex();
103
+ })());
104
+ };
105
+ onArchivistInserted = () => {
85
106
  forget(this.updateIndex());
86
107
  };
87
108
  async stopHandler(_timeout) {
88
109
  const archivist = await this.archivistInstance(true);
89
- archivist.off("inserted", this.onArchivistInsert);
110
+ archivist.off("inserted", this.onArchivistInserted);
111
+ archivist.off("deleted", this.onArchivistDeleted);
112
+ archivist.off("cleared", this.onArchivistCleared);
90
113
  return await super.stopHandler();
91
114
  }
92
115
  //index any new payloads
package/dist/browser/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SAASC,oCAA6F;AAEtG,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,eAAAA;EA5BV,OA4BUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,cAAM2B,aAAa,KAAK1D;AACxB,aAAKA,eAAe,MAAM2D,eAAe7C,KAAKgB,SAAS0B,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAK5D,iBAAiB0D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA5B,iBAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK0D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc+D,cAAcrC,UAA2C;AACrE,UAAMf,QAAQ,MAAMgD,eAAeK,UAAUtC,QAAAA;AAC7C,SAAK5B,aAAamE,KAAI,GAAItD,KAAAA;AAC1B,WAAOmB,SAASnB,MAAMiD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push"]}
1
+ {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInserted)\n archivist?.on('cleared', this.onArchivistCleared)\n archivist?.on('deleted', this.onArchivistDeleted)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected async clearIndex() {\n await this._updatePayloadPairsMutex.runExclusive(() => {\n this._indexOffset = undefined\n this.payloadPairs = []\n })\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInserted)\n archivist.off('deleted', this.onArchivistDeleted)\n archivist.off('cleared', this.onArchivistCleared)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SAASC,oCAA6F;AAEtG,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,eAAAA;EA5BV,OA4BUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK3B,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAgB4B,aAAa;AAC3B,UAAM,KAAK1B,yBAAyB2B,aAAa,MAAA;AAC/C,WAAK5B,eAAe6B;AACpB,WAAK/B,eAAe,CAAA;IACtB,CAAA;EACF;EAEA,MAAyBgC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKrC,yBAAyB2B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAO4D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK9D;AACxB,aAAKA,eAAe,MAAM+D,eAAejD,KAAKqB,SAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKhE,iBAAiB8D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,iBAAS,KAAKrC,aAAasC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcmE,cAAcpC,UAA2C;AACrE,UAAMpB,QAAQ,MAAMoD,eAAeK,UAAUrC,QAAAA;AAC7C,SAAKjC,aAAauE,KAAI,GAAI1D,KAAAA;AAC1B,WAAOwB,SAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push"]}
package/dist/neutral/Diviner.d.cts CHANGED
@@ -21,8 +21,11 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
21
21
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
22
22
  protected archivistInstance(): Promise<ArchivistInstance | undefined>;
23
23
  protected archivistInstance(required: true): Promise<ArchivistInstance>;
24
+ protected clearIndex(): Promise<void>;
24
25
  protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
25
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']>;
26
+ protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']>;
27
+ protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']>;
28
+ protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']>;
26
29
  protected stopHandler(_timeout?: number | undefined): Promise<boolean>;
27
30
  protected updateIndex(): Promise<void>;
28
31
  private indexPayloads;
package/dist/neutral/Diviner.d.cts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAetE,UAAU;cAOD,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAGD,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAED,SAAS,CAAC,mBAAmB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAEjF;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cASrE,WAAW;YAiBb,aAAa;CAK5B"}
package/dist/neutral/Diviner.d.mts CHANGED
@@ -21,8 +21,11 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
21
21
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
22
22
  protected archivistInstance(): Promise<ArchivistInstance | undefined>;
23
23
  protected archivistInstance(required: true): Promise<ArchivistInstance>;
24
+ protected clearIndex(): Promise<void>;
24
25
  protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
25
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']>;
26
+ protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']>;
27
+ protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']>;
28
+ protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']>;
26
29
  protected stopHandler(_timeout?: number | undefined): Promise<boolean>;
27
30
  protected updateIndex(): Promise<void>;
28
31
  private indexPayloads;
package/dist/neutral/Diviner.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAetE,UAAU;cAOD,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAGD,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAED,SAAS,CAAC,mBAAmB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAEjF;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cASrE,WAAW;YAiBb,aAAa;CAK5B"}
package/dist/neutral/Diviner.d.ts CHANGED
@@ -21,8 +21,11 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
21
21
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
22
22
  protected archivistInstance(): Promise<ArchivistInstance | undefined>;
23
23
  protected archivistInstance(required: true): Promise<ArchivistInstance>;
24
+ protected clearIndex(): Promise<void>;
24
25
  protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
25
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']>;
26
+ protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']>;
27
+ protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']>;
28
+ protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']>;
26
29
  protected stopHandler(_timeout?: number | undefined): Promise<boolean>;
27
30
  protected updateIndex(): Promise<void>;
28
31
  private indexPayloads;
package/dist/neutral/Diviner.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAetE,UAAU;cAOD,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAGD,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAED,SAAS,CAAC,mBAAmB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAEjF;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cASrE,WAAW;YAiBb,aAAa;CAK5B"}
package/dist/neutral/index.cjs CHANGED
@@ -76,11 +76,19 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
76
76
  if (required && !archivist) {
77
77
  throw new Error("Failed to find archivist");
78
78
  }
79
- archivist?.on("inserted", this.onArchivistInsert);
79
+ archivist?.on("inserted", this.onArchivistInserted);
80
+ archivist?.on("cleared", this.onArchivistCleared);
81
+ archivist?.on("deleted", this.onArchivistDeleted);
80
82
  this._archivistInstance = archivist;
81
83
  }
82
84
  return this._archivistInstance;
83
85
  }
86
+ async clearIndex() {
87
+ await this._updatePayloadPairsMutex.runExclusive(() => {
88
+ this._indexOffset = void 0;
89
+ this.payloadPairs = [];
90
+ });
91
+ }
84
92
  async divineHandler(payloads) {
85
93
  const filters = payloads?.filter(import_diviner_payload_model.isPayloadDivinerQueryPayload) ?? [];
86
94
  (0, import_assert.assertEx)(filters.length < 2, () => "Multiple PayloadDivinerQuery payloads may not be specified");
@@ -106,12 +114,27 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
106
114
  throw new Error('Archivist does not support "all"');
107
115
  }
108
116
  }
109
- onArchivistInsert = () => {
117
+ onArchivistCleared = () => {
118
+ (0, import_forget.forget)((async () => {
119
+ await this.clearIndex();
120
+ await this.updateIndex();
121
+ })());
122
+ };
123
+ // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
124
+ onArchivistDeleted = () => {
125
+ (0, import_forget.forget)((async () => {
126
+ await this.clearIndex();
127
+ await this.updateIndex();
128
+ })());
129
+ };
130
+ onArchivistInserted = () => {
110
131
  (0, import_forget.forget)(this.updateIndex());
111
132
  };
112
133
  async stopHandler(_timeout) {
113
134
  const archivist = await this.archivistInstance(true);
114
- archivist.off("inserted", this.onArchivistInsert);
135
+ archivist.off("inserted", this.onArchivistInserted);
136
+ archivist.off("deleted", this.onArchivistDeleted);
137
+ archivist.off("cleared", this.onArchivistCleared);
115
138
  return await super.stopHandler();
116
139
  }
117
140
  //index any new payloads
package/dist/neutral/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAAsG;AAEtG,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,+CAAAA;EA5BV,OA4BUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,cAAM2B,aAAa,KAAK1D;AACxB,aAAKA,eAAe,MAAM2D,sCAAe7C,SAAKgB,wBAAS0B,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAK5D,iBAAiB0D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA5B,oCAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK0D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc+D,cAAcrC,UAA2C;AACrE,UAAMf,QAAQ,MAAMgD,sCAAeK,UAAUtC,QAAAA;AAC7C,SAAK5B,aAAamE,KAAI,GAAItD,KAAAA;AAC1B,eAAOmB,wBAASnB,MAAMiD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push"]}
1
+ {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInserted)\n archivist?.on('cleared', this.onArchivistCleared)\n archivist?.on('deleted', this.onArchivistDeleted)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected async clearIndex() {\n await this._updatePayloadPairsMutex.runExclusive(() => {\n this._indexOffset = undefined\n this.payloadPairs = []\n })\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInserted)\n archivist.off('deleted', this.onArchivistDeleted)\n archivist.off('cleared', this.onArchivistCleared)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAAsG;AAEtG,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,+CAAAA;EA5BV,OA4BUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK3B,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAgB4B,aAAa;AAC3B,UAAM,KAAK1B,yBAAyB2B,aAAa,MAAA;AAC/C,WAAK5B,eAAe6B;AACpB,WAAK/B,eAAe,CAAA;IACtB,CAAA;EACF;EAEA,MAAyBgC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKrC,yBAAyB2B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAO4D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK9D;AACxB,aAAKA,eAAe,MAAM+D,sCAAejD,SAAKqB,wBAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKhE,iBAAiB8D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,oCAAS,KAAKrC,aAAasC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcmE,cAAcpC,UAA2C;AACrE,UAAMpB,QAAQ,MAAMoD,sCAAeK,UAAUrC,QAAAA;AAC7C,SAAKjC,aAAauE,KAAI,GAAI1D,KAAAA;AAC1B,eAAOwB,wBAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push"]}
package/dist/neutral/index.js CHANGED
@@ -51,11 +51,19 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
51
51
  if (required && !archivist) {
52
52
  throw new Error("Failed to find archivist");
53
53
  }
54
- archivist?.on("inserted", this.onArchivistInsert);
54
+ archivist?.on("inserted", this.onArchivistInserted);
55
+ archivist?.on("cleared", this.onArchivistCleared);
56
+ archivist?.on("deleted", this.onArchivistDeleted);
55
57
  this._archivistInstance = archivist;
56
58
  }
57
59
  return this._archivistInstance;
58
60
  }
61
+ async clearIndex() {
62
+ await this._updatePayloadPairsMutex.runExclusive(() => {
63
+ this._indexOffset = void 0;
64
+ this.payloadPairs = [];
65
+ });
66
+ }
59
67
  async divineHandler(payloads) {
60
68
  const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? [];
61
69
  assertEx(filters.length < 2, () => "Multiple PayloadDivinerQuery payloads may not be specified");
@@ -81,12 +89,27 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
81
89
  throw new Error('Archivist does not support "all"');
82
90
  }
83
91
  }
84
- onArchivistInsert = () => {
92
+ onArchivistCleared = () => {
93
+ forget((async () => {
94
+ await this.clearIndex();
95
+ await this.updateIndex();
96
+ })());
97
+ };
98
+ // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
99
+ onArchivistDeleted = () => {
100
+ forget((async () => {
101
+ await this.clearIndex();
102
+ await this.updateIndex();
103
+ })());
104
+ };
105
+ onArchivistInserted = () => {
85
106
  forget(this.updateIndex());
86
107
  };
87
108
  async stopHandler(_timeout) {
88
109
  const archivist = await this.archivistInstance(true);
89
- archivist.off("inserted", this.onArchivistInsert);
110
+ archivist.off("inserted", this.onArchivistInserted);
111
+ archivist.off("deleted", this.onArchivistDeleted);
112
+ archivist.off("cleared", this.onArchivistCleared);
90
113
  return await super.stopHandler();
91
114
  }
92
115
  //index any new payloads
package/dist/neutral/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SAASC,oCAA6F;AAEtG,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,eAAAA;EA5BV,OA4BUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,cAAM2B,aAAa,KAAK1D;AACxB,aAAKA,eAAe,MAAM2D,eAAe7C,KAAKgB,SAAS0B,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAK5D,iBAAiB0D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA5B,iBAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK0D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc+D,cAAcrC,UAA2C;AACrE,UAAMf,QAAQ,MAAMgD,eAAeK,UAAUtC,QAAAA;AAC7C,SAAK5B,aAAamE,KAAI,GAAItD,KAAAA;AAC1B,WAAOmB,SAASnB,MAAMiD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push"]}
1
+ {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInserted)\n archivist?.on('cleared', this.onArchivistCleared)\n archivist?.on('deleted', this.onArchivistDeleted)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected async clearIndex() {\n await this._updatePayloadPairsMutex.runExclusive(() => {\n this._indexOffset = undefined\n this.payloadPairs = []\n })\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInserted)\n archivist.off('deleted', this.onArchivistDeleted)\n archivist.off('cleared', this.onArchivistCleared)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SAASC,oCAA6F;AAEtG,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,wBAAN,cASGC,eAAAA;EA5BV,OA4BUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBZ;EACvC;EAEA,IAAcc,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK3B,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAgB4B,aAAa;AAC3B,UAAM,KAAK1B,yBAAyB2B,aAAa,MAAA;AAC/C,WAAK5B,eAAe6B;AACpB,WAAK/B,eAAe,CAAA;IACtB,CAAA;EACF;EAEA,MAAyBgC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKrC,yBAAyB2B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAO4D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK9D;AACxB,aAAKA,eAAe,MAAM+D,eAAejD,KAAKqB,SAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKhE,iBAAiB8D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,iBAAS,KAAKrC,aAAasC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcmE,cAAcpC,UAA2C;AACrE,UAAMpB,QAAQ,MAAMoD,eAAeK,UAAUrC,QAAAA;AAC7C,SAAKjC,aAAauE,KAAI,GAAI1D,KAAAA;AAC1B,WAAOwB,SAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push"]}
package/dist/node/Diviner.d.cts CHANGED
@@ -21,8 +21,11 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
21
21
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
22
22
  protected archivistInstance(): Promise<ArchivistInstance | undefined>;
23
23
  protected archivistInstance(required: true): Promise<ArchivistInstance>;
24
+ protected clearIndex(): Promise<void>;
24
25
  protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
25
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']>;
26
+ protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']>;
27
+ protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']>;
28
+ protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']>;
26
29
  protected stopHandler(_timeout?: number | undefined): Promise<boolean>;
27
30
  protected updateIndex(): Promise<void>;
28
31
  private indexPayloads;
package/dist/node/Diviner.d.cts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAetE,UAAU;cAOD,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAGD,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAED,SAAS,CAAC,mBAAmB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAEjF;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cASrE,WAAW;YAiBb,aAAa;CAK5B"}
package/dist/node/Diviner.d.mts CHANGED
@@ -21,8 +21,11 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
21
21
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
22
22
  protected archivistInstance(): Promise<ArchivistInstance | undefined>;
23
23
  protected archivistInstance(required: true): Promise<ArchivistInstance>;
24
+ protected clearIndex(): Promise<void>;
24
25
  protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
25
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']>;
26
+ protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']>;
27
+ protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']>;
28
+ protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']>;
26
29
  protected stopHandler(_timeout?: number | undefined): Promise<boolean>;
27
30
  protected updateIndex(): Promise<void>;
28
31
  private indexPayloads;
package/dist/node/Diviner.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAetE,UAAU;cAOD,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAGD,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAED,SAAS,CAAC,mBAAmB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAEjF;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cASrE,WAAW;YAiBb,aAAa;CAK5B"}
package/dist/node/Diviner.d.ts CHANGED
@@ -21,8 +21,11 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
21
21
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
22
22
  protected archivistInstance(): Promise<ArchivistInstance | undefined>;
23
23
  protected archivistInstance(required: true): Promise<ArchivistInstance>;
24
+ protected clearIndex(): Promise<void>;
24
25
  protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
25
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']>;
26
+ protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']>;
27
+ protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']>;
28
+ protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']>;
26
29
  protected stopHandler(_timeout?: number | undefined): Promise<boolean>;
27
30
  protected updateIndex(): Promise<void>;
28
31
  private indexPayloads;
package/dist/node/Diviner.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAiBb,aAAa;CAK5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAgC,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAA;AAC1I,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAMtE,eAAO,MAAM,iCAAiC,8CAAwD,CAAA;AACtG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA8D;IAC9G,gBAAyB,mBAAmB,EAAE,MAAM,CAAoC;IAExF,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAc,EAAE,MAAM,CAAC,EAAE,IAAI;IAIlD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAetE,UAAU;cAOD,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAGD,SAAS,CAAC,kBAAkB,EAAE,aAAa,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAO/E;IAED,SAAS,CAAC,mBAAmB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAEjF;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cASrE,WAAW;YAiBb,aAAa;CAK5B"}
package/dist/node/index.cjs CHANGED
@@ -76,11 +76,19 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
76
76
  if (required && !archivist) {
77
77
  throw new Error("Failed to find archivist");
78
78
  }
79
- archivist == null ? void 0 : archivist.on("inserted", this.onArchivistInsert);
79
+ archivist == null ? void 0 : archivist.on("inserted", this.onArchivistInserted);
80
+ archivist == null ? void 0 : archivist.on("cleared", this.onArchivistCleared);
81
+ archivist == null ? void 0 : archivist.on("deleted", this.onArchivistDeleted);
80
82
  this._archivistInstance = archivist;
81
83
  }
82
84
  return this._archivistInstance;
83
85
  }
86
+ async clearIndex() {
87
+ await this._updatePayloadPairsMutex.runExclusive(() => {
88
+ this._indexOffset = void 0;
89
+ this.payloadPairs = [];
90
+ });
91
+ }
84
92
  async divineHandler(payloads) {
85
93
  const filters = (payloads == null ? void 0 : payloads.filter(import_diviner_payload_model.isPayloadDivinerQueryPayload)) ?? [];
86
94
  (0, import_assert.assertEx)(filters.length < 2, () => "Multiple PayloadDivinerQuery payloads may not be specified");
@@ -107,12 +115,27 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
107
115
  throw new Error('Archivist does not support "all"');
108
116
  }
109
117
  }
110
- onArchivistInsert = () => {
118
+ onArchivistCleared = () => {
119
+ (0, import_forget.forget)((async () => {
120
+ await this.clearIndex();
121
+ await this.updateIndex();
122
+ })());
123
+ };
124
+ // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
125
+ onArchivistDeleted = () => {
126
+ (0, import_forget.forget)((async () => {
127
+ await this.clearIndex();
128
+ await this.updateIndex();
129
+ })());
130
+ };
131
+ onArchivistInserted = () => {
111
132
  (0, import_forget.forget)(this.updateIndex());
112
133
  };
113
134
  async stopHandler(_timeout) {
114
135
  const archivist = await this.archivistInstance(true);
115
- archivist.off("inserted", this.onArchivistInsert);
136
+ archivist.off("inserted", this.onArchivistInserted);
137
+ archivist.off("deleted", this.onArchivistDeleted);
138
+ archivist.off("cleared", this.onArchivistCleared);
116
139
  return await super.stopHandler();
117
140
  }
118
141
  //index any new payloads
package/dist/node/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAAsG;AAEtG,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,yBAAN,MAAMA,+BASHC,+CAAAA;EAIEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEUY,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,+DAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAjG9B;AAkGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,cAAM2B,aAAa,KAAK1D;AACxB,aAAKA,eAAe,MAAM2D,sCAAe7C,SAAKgB,wBAAS0B,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAK5D,iBAAiB0D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA5B,oCAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK0D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc+D,cAAcrC,UAA2C;AACrE,UAAMf,QAAQ,MAAMgD,sCAAeK,UAAUtC,QAAAA;AAC7C,SAAK5B,aAAamE,KAAI,GAAItD,KAAAA;AAC1B,eAAOmB,wBAASnB,MAAMiD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAtHU/D;AACR,cAVWD,wBAUcsE,iBAA0B;KAAI,2DAAMA;EAAevE;;AAC5E,cAXWC,wBAWcuE,uBAA8BxE;AAXlD,IAAMC,wBAAN;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","configSchemas","defaultConfigSchema"]}
1
+ {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInserted)\n archivist?.on('cleared', this.onArchivistCleared)\n archivist?.on('deleted', this.onArchivistDeleted)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected async clearIndex() {\n await this._updatePayloadPairsMutex.runExclusive(() => {\n this._indexOffset = undefined\n this.payloadPairs = []\n })\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInserted)\n archivist.off('deleted', this.onArchivistDeleted)\n archivist.off('cleared', this.onArchivistCleared)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAAsG;AAEtG,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,yBAAN,MAAMA,+BASHC,+CAAAA;EAIEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEUY,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/BH,6CAAWE,GAAG,WAAW,KAAKE;AAC9BJ,6CAAWE,GAAG,WAAW,KAAKG;AAC9B,WAAK3B,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAgB4B,aAAa;AAC3B,UAAM,KAAK1B,yBAAyB2B,aAAa,MAAA;AAC/C,WAAK5B,eAAe6B;AACpB,WAAK/B,eAAe,CAAA;IACtB,CAAA;EACF;EAEA,MAAyBgC,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,+DAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,mCAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AA1G9B;AA2GkB,kBAAML,QAAOjC,mCAAUkC;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,aAAYA,mCAAUkC,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKrC,yBAAyB2B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAO4D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK9D;AACxB,aAAKA,eAAe,MAAM+D,sCAAejD,SAAKqB,wBAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKhE,iBAAiB8D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,oCAAS,KAAKrC,aAAasC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcmE,cAAcpC,UAA2C;AACrE,UAAMpB,QAAQ,MAAMoD,sCAAeK,UAAUrC,QAAAA;AAC7C,SAAKjC,aAAauE,KAAI,GAAI1D,KAAAA;AAC1B,eAAOwB,wBAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAnJUnE;AACR,cAVWD,wBAUc0E,iBAA0B;KAAI,2DAAMA;EAAe3E;;AAC5E,cAXWC,wBAWc2E,uBAA8B5E;AAXlD,IAAMC,wBAAN;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","configSchemas","defaultConfigSchema"]}
package/dist/node/index.js CHANGED
@@ -51,11 +51,19 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
51
51
  if (required && !archivist) {
52
52
  throw new Error("Failed to find archivist");
53
53
  }
54
- archivist == null ? void 0 : archivist.on("inserted", this.onArchivistInsert);
54
+ archivist == null ? void 0 : archivist.on("inserted", this.onArchivistInserted);
55
+ archivist == null ? void 0 : archivist.on("cleared", this.onArchivistCleared);
56
+ archivist == null ? void 0 : archivist.on("deleted", this.onArchivistDeleted);
55
57
  this._archivistInstance = archivist;
56
58
  }
57
59
  return this._archivistInstance;
58
60
  }
61
+ async clearIndex() {
62
+ await this._updatePayloadPairsMutex.runExclusive(() => {
63
+ this._indexOffset = void 0;
64
+ this.payloadPairs = [];
65
+ });
66
+ }
59
67
  async divineHandler(payloads) {
60
68
  const filters = (payloads == null ? void 0 : payloads.filter(isPayloadDivinerQueryPayload)) ?? [];
61
69
  assertEx(filters.length < 2, () => "Multiple PayloadDivinerQuery payloads may not be specified");
@@ -82,12 +90,27 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
82
90
  throw new Error('Archivist does not support "all"');
83
91
  }
84
92
  }
85
- onArchivistInsert = () => {
93
+ onArchivistCleared = () => {
94
+ forget((async () => {
95
+ await this.clearIndex();
96
+ await this.updateIndex();
97
+ })());
98
+ };
99
+ // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
100
+ onArchivistDeleted = () => {
101
+ forget((async () => {
102
+ await this.clearIndex();
103
+ await this.updateIndex();
104
+ })());
105
+ };
106
+ onArchivistInserted = () => {
86
107
  forget(this.updateIndex());
87
108
  };
88
109
  async stopHandler(_timeout) {
89
110
  const archivist = await this.archivistInstance(true);
90
- archivist.off("inserted", this.onArchivistInsert);
111
+ archivist.off("inserted", this.onArchivistInserted);
112
+ archivist.off("deleted", this.onArchivistDeleted);
113
+ archivist.off("cleared", this.onArchivistCleared);
91
114
  return await super.stopHandler();
92
115
  }
93
116
  //index any new payloads
package/dist/node/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SAASC,oCAA6F;AAEtG,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,yBAAN,MAAMA,+BASHC,eAAAA;EAIEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEUY,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,kCAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAjG9B;AAkGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,cAAM2B,aAAa,KAAK1D;AACxB,aAAKA,eAAe,MAAM2D,eAAe7C,KAAKgB,SAAS0B,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAK5D,iBAAiB0D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA5B,iBAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK0D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc+D,cAAcrC,UAA2C;AACrE,UAAMf,QAAQ,MAAMgD,eAAeK,UAAUtC,QAAAA;AAC7C,SAAK5B,aAAamE,KAAI,GAAItD,KAAAA;AAC1B,WAAOmB,SAASnB,MAAMiD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAtHU/D;AACR,cAVWD,wBAUcsE,iBAA0B;KAAI,2DAAMA;EAAevE;;AAC5E,cAXWC,wBAWcuE,uBAA8BxE;AAXlD,IAAMC,wBAAN;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","configSchemas","defaultConfigSchema"]}
1
+ {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport { isPayloadDivinerQueryPayload, Order, PayloadDivinerParams, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'desc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = [...this.payloadPairs].reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return pairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInserted)\n archivist?.on('cleared', this.onArchivistCleared)\n archivist?.on('deleted', this.onArchivistDeleted)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected async clearIndex() {\n await this._updatePayloadPairsMutex.runExclusive(() => {\n this._indexOffset = undefined\n this.payloadPairs = []\n })\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n forget(\n (async () => {\n await this.clearIndex()\n await this.updateIndex()\n })(),\n )\n }\n\n protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInserted)\n archivist.off('deleted', this.onArchivistDeleted)\n archivist.off('cleared', this.onArchivistCleared)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n const prevOffset = this._indexOffset\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n if (this._indexOffset === prevOffset) {\n this.logger.warn('next offset not found', prevOffset)\n }\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SAASC,oCAA6F;AAEtG,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAG1C,IAAMC,yBAAN,MAAMA,+BASHC,eAAAA;EAIEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEUY,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKb;MAAcoB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/BH,6CAAWE,GAAG,WAAW,KAAKE;AAC9BJ,6CAAWE,GAAG,WAAW,KAAKG;AAC9B,WAAK3B,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAgB4B,aAAa;AAC3B,UAAM,KAAK1B,yBAAyB2B,aAAa,MAAA;AAC/C,WAAK5B,eAAe6B;AACpB,WAAK/B,eAAe,CAAA;IACtB,CAAA;EACF;EAEA,MAAyBgC,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,kCAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,mCAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AA1G9B;AA2GkB,kBAAML,QAAOjC,mCAAUkC;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,aAAYA,mCAAUkC,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKrC,yBAAyB2B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAO4D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK9D;AACxB,aAAKA,eAAe,MAAM+D,eAAejD,KAAKqB,SAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKhE,iBAAiB8D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,iBAAS,KAAKrC,aAAasC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcmE,cAAcpC,UAA2C;AACrE,UAAMpB,QAAQ,MAAMoD,eAAeK,UAAUrC,QAAAA;AAC7C,SAAKjC,aAAauE,KAAI,GAAI1D,KAAAA;AAC1B,WAAOwB,SAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAnJUnE;AACR,cAVWD,wBAUc0E,iBAA0B;KAAI,2DAAMA;EAAe3E;;AAC5E,cAXWC,wBAWc2E,uBAA8B5E;AAXlD,IAAMC,wBAAN;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","configSchemas","defaultConfigSchema"]}
package/package.json CHANGED
@@ -14,22 +14,22 @@
14
14
  "@xylabs/forget": "^3.3.2",
15
15
  "@xylabs/hex": "^3.3.2",
16
16
  "@xylabs/object": "^3.3.2",
17
- "@xyo-network/archivist-model": "~2.100.0",
18
- "@xyo-network/diviner-model": "~2.100.0",
19
- "@xyo-network/diviner-payload-abstract": "~2.100.0",
20
- "@xyo-network/diviner-payload-model": "~2.100.0",
21
- "@xyo-network/module-events": "~2.100.0",
22
- "@xyo-network/payload-builder": "~2.100.0",
23
- "@xyo-network/payload-model": "~2.100.0",
17
+ "@xyo-network/archivist-model": "~2.100.2",
18
+ "@xyo-network/diviner-model": "~2.100.2",
19
+ "@xyo-network/diviner-payload-abstract": "~2.100.2",
20
+ "@xyo-network/diviner-payload-model": "~2.100.2",
21
+ "@xyo-network/module-events": "~2.100.2",
22
+ "@xyo-network/payload-builder": "~2.100.2",
23
+ "@xyo-network/payload-model": "~2.100.2",
24
24
  "async-mutex": "^0.5.0"
25
25
  },
26
26
  "devDependencies": {
27
27
  "@xylabs/ts-scripts-yarn3": "^3.10.0",
28
28
  "@xylabs/tsconfig": "^3.10.0",
29
- "@xyo-network/account": "~2.100.0",
30
- "@xyo-network/archivist-memory": "~2.100.0",
31
- "@xyo-network/node-memory": "~2.100.0",
32
- "@xyo-network/payload-builder": "~2.100.0",
29
+ "@xyo-network/account": "~2.100.2",
30
+ "@xyo-network/archivist-memory": "~2.100.2",
31
+ "@xyo-network/node-memory": "~2.100.2",
32
+ "@xyo-network/payload-builder": "~2.100.2",
33
33
  "typescript": "^5.4.5"
34
34
  },
35
35
  "description": "Primary SDK for using XYO Protocol 2.0",
@@ -71,6 +71,6 @@
71
71
  "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
72
72
  },
73
73
  "sideEffects": false,
74
- "version": "2.100.0",
74
+ "version": "2.100.2",
75
75
  "type": "module"
76
76
  }
package/src/Diviner.ts CHANGED
@@ -68,12 +68,21 @@ export class GenericPayloadDiviner<
68
68
  if (required && !archivist) {
69
69
  throw new Error('Failed to find archivist')
70
70
  }
71
- archivist?.on('inserted', this.onArchivistInsert)
71
+ archivist?.on('inserted', this.onArchivistInserted)
72
+ archivist?.on('cleared', this.onArchivistCleared)
73
+ archivist?.on('deleted', this.onArchivistDeleted)
72
74
  this._archivistInstance = archivist
73
75
  }
74
76
  return this._archivistInstance
75
77
  }
76
78
 
79
+ protected async clearIndex() {
80
+ await this._updatePayloadPairsMutex.runExclusive(() => {
81
+ this._indexOffset = undefined
82
+ this.payloadPairs = []
83
+ })
84
+ }
85
+
77
86
  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {
78
87
  const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []
79
88
  assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')
@@ -110,14 +119,34 @@ export class GenericPayloadDiviner<
110
119
  }
111
120
  }
112
121
 
113
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {
114
- //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))
122
+ protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {
123
+ forget(
124
+ (async () => {
125
+ await this.clearIndex()
126
+ await this.updateIndex()
127
+ })(),
128
+ )
129
+ }
130
+
131
+ // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
132
+ protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {
133
+ forget(
134
+ (async () => {
135
+ await this.clearIndex()
136
+ await this.updateIndex()
137
+ })(),
138
+ )
139
+ }
140
+
141
+ protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {
115
142
  forget(this.updateIndex())
116
143
  }
117
144
 
118
145
  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {
119
146
  const archivist = await this.archivistInstance(true)
120
- archivist.off('inserted', this.onArchivistInsert)
147
+ archivist.off('inserted', this.onArchivistInserted)
148
+ archivist.off('deleted', this.onArchivistDeleted)
149
+ archivist.off('cleared', this.onArchivistCleared)
121
150
  return await super.stopHandler()
122
151
  }
123
152