@xyo-network/diviner-payload-generic 2.95.6 → 2.95.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. package/dist/browser/Diviner.d.cts +1 -1
  2. package/dist/browser/Diviner.d.cts.map +1 -1
  3. package/dist/browser/Diviner.d.mts +1 -1
  4. package/dist/browser/Diviner.d.mts.map +1 -1
  5. package/dist/browser/Diviner.d.ts +1 -1
  6. package/dist/browser/Diviner.d.ts.map +1 -1
  7. package/dist/browser/index.cjs +6 -7
  8. package/dist/browser/index.cjs.map +1 -1
  9. package/dist/browser/index.js +6 -7
  10. package/dist/browser/index.js.map +1 -1
  11. package/dist/node/Diviner.d.cts +1 -1
  12. package/dist/node/Diviner.d.cts.map +1 -1
  13. package/dist/node/Diviner.d.mts +1 -1
  14. package/dist/node/Diviner.d.mts.map +1 -1
  15. package/dist/node/Diviner.d.ts +1 -1
  16. package/dist/node/Diviner.d.ts.map +1 -1
  17. package/dist/node/index.cjs +6 -8
  18. package/dist/node/index.cjs.map +1 -1
  19. package/dist/node/index.js +6 -8
  20. package/dist/node/index.js.map +1 -1
  21. package/package.json +18 -18
  22. package/src/Diviner.ts +7 -8
package/dist/browser/Diviner.d.cts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/browser/Diviner.d.cts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAab,aAAa;CAI5B"}
package/dist/browser/Diviner.d.mts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/browser/Diviner.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAab,aAAa;CAI5B"}
package/dist/browser/Diviner.d.ts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/browser/Diviner.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAab,aAAa;CAI5B"}
package/dist/browser/index.cjs CHANGED
@@ -43,6 +43,7 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
43
43
  ];
44
44
  payloadPairs = [];
45
45
  _archivistInstance;
46
+ _indexOffset;
46
47
  _updatePayloadPairsMutex = new import_async_mutex.Mutex();
47
48
  get indexBatchSize() {
48
49
  return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE;
@@ -50,9 +51,6 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
50
51
  get maxIndexSize() {
51
52
  return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE;
52
53
  }
53
- get payloadPairsOffset() {
54
- return this.payloadPairs.at(-1)?.[1];
55
- }
56
54
  all(order = "asc", offset) {
57
55
  return order === "asc" ? this.allAsc(offset) : this.allDesc(offset);
58
56
  }
@@ -102,8 +100,8 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
102
100
  throw new Error('Archivist does not support "all"');
103
101
  }
104
102
  }
105
- onArchivistInsert = ({ payloads }) => {
106
- (0, import_forget.forget)(this.indexPayloads(payloads));
103
+ onArchivistInsert = () => {
104
+ (0, import_forget.forget)(this.updateIndex());
107
105
  };
108
106
  async stopHandler(_timeout) {
109
107
  const archivist = await this.archivistInstance(true);
@@ -116,14 +114,15 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
116
114
  const archivist = await this.archivistInstance(true);
117
115
  let newPayloads = await archivist.next({
118
116
  limit: 100,
119
- offset: this.payloadPairsOffset
117
+ offset: this._indexOffset
120
118
  });
121
119
  while (newPayloads.length > 0) {
120
+ this._indexOffset = await import_payload_builder.PayloadBuilder.hash((0, import_assert.assertEx)(newPayloads.at(-1)));
122
121
  (0, import_assert.assertEx)(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
123
122
  await this.indexPayloads(newPayloads);
124
123
  newPayloads = await archivist.next({
125
124
  limit: 100,
126
- offset: this.payloadPairsOffset
125
+ offset: this._indexOffset
127
126
  });
128
127
  }
129
128
  });
package/dist/browser/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected get payloadPairsOffset() {\n return this.payloadPairs.at(-1)?.[1]\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {\n forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,wBAAN,cASGC,+CAAAA;EA/BV,OA+BUA;;;EACR,OAAgBC,gBAAgB;IAACC;;EAEvBC,eAAyC,CAAA;EAE3CC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEA,IAAcY,qBAAqB;AACjC,WAAO,KAAKP,aAAaQ,GAAG,EAAC,IAAK,CAAA;EACpC;EAEUC,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKd,aAAaqB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAK1B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAyB2B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,CAAC,EAAEE,SAAQ,MAAE;AAC9FyB,8BAAO,KAAKC,cAAc1B,QAAAA,CAAAA;EAC5B;EAEA,MAAyB2B,YAAYC,UAAiD;AACpF,UAAMjC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUkC,IAAI,YAAY,KAAK/B,iBAAiB;AAChD,WAAO,MAAM,MAAM6B,YAAAA;EACrB;;EAGA,MAAgBpB,cAAc;AAC5B,UAAM,KAAKlC,yBAAyByD,aAAa,YAAA;AAC/C,YAAMnC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIsC,cAAe,MAAMpC,UAAUqC,KAAK;QAAEpB,OAAO;QAAK9B,QAAQ,KAAKJ;MAAmB,CAAA;AACtF,aAAOqD,YAAY1B,SAAS,GAAG;AAC7BD,oCAAS,KAAKjC,aAAakC,SAAS0B,YAAY1B,UAAU,KAAK5B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKiD,cAAcK,WAAAA;AACzBA,sBAAe,MAAMpC,UAAUqC,KAAK;UAAEpB,OAAO;UAAK9B,QAAQ,KAAKJ;QAAmB,CAAA;MACpF;IACF,CAAA;EACF;EAEA,MAAcgD,cAAc1B,UAA4B;AACtD,UAAMf,QAAQ,MAAMgD,sCAAeC,UAAUlC,QAAAA;AAC7C,SAAK7B,aAAagE,KAAI,GAAIlD,KAAAA;EAC5B;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","configSchemas","PayloadDivinerConfigSchema","payloadPairs","_archivistInstance","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","payloadPairsOffset","at","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","indexPayloads","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","hashPairs","push"]}
1
+ {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,wBAAN,cASGC,+CAAAA;EA/BV,OA+BUA;;;EACR,OAAgBC,gBAAgB;IAACC;;EAEvBC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBX;EACvC;EAEA,IAAca,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBZ;EACrC;EAEUa,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,aAAK/B,eAAe,MAAM0D,sCAAe5C,SAAKgB,wBAAS0B,YAAYG,GAAG,EAAC,CAAA,CAAA;AACvE7B,oCAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKuD,cAAcJ,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc4D,cAAclC,UAA4B;AACtD,UAAMf,QAAQ,MAAM+C,sCAAeG,UAAUnC,QAAAA;AAC7C,SAAK5B,aAAagE,KAAI,GAAInD,KAAAA;EAC5B;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","configSchemas","PayloadDivinerConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","at","indexPayloads","hashPairs","push"]}
package/dist/browser/index.js CHANGED
@@ -19,6 +19,7 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
19
19
  ];
20
20
  payloadPairs = [];
21
21
  _archivistInstance;
22
+ _indexOffset;
22
23
  _updatePayloadPairsMutex = new Mutex();
23
24
  get indexBatchSize() {
24
25
  return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE;
@@ -26,9 +27,6 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
26
27
  get maxIndexSize() {
27
28
  return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE;
28
29
  }
29
- get payloadPairsOffset() {
30
- return this.payloadPairs.at(-1)?.[1];
31
- }
32
30
  all(order = "asc", offset) {
33
31
  return order === "asc" ? this.allAsc(offset) : this.allDesc(offset);
34
32
  }
@@ -78,8 +76,8 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
78
76
  throw new Error('Archivist does not support "all"');
79
77
  }
80
78
  }
81
- onArchivistInsert = ({ payloads }) => {
82
- forget(this.indexPayloads(payloads));
79
+ onArchivistInsert = () => {
80
+ forget(this.updateIndex());
83
81
  };
84
82
  async stopHandler(_timeout) {
85
83
  const archivist = await this.archivistInstance(true);
@@ -92,14 +90,15 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
92
90
  const archivist = await this.archivistInstance(true);
93
91
  let newPayloads = await archivist.next({
94
92
  limit: 100,
95
- offset: this.payloadPairsOffset
93
+ offset: this._indexOffset
96
94
  });
97
95
  while (newPayloads.length > 0) {
96
+ this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)));
98
97
  assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
99
98
  await this.indexPayloads(newPayloads);
100
99
  newPayloads = await archivist.next({
101
100
  limit: 100,
102
- offset: this.payloadPairsOffset
101
+ offset: this._indexOffset
103
102
  });
104
103
  }
105
104
  });
package/dist/browser/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected get payloadPairsOffset() {\n return this.payloadPairs.at(-1)?.[1]\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {\n forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,8BAEAC,kCAGK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,wBAAN,cASGC,eAAAA;EA/BV,OA+BUA;;;EACR,OAAgBC,gBAAgB;IAACC;;EAEvBC,eAAyC,CAAA;EAE3CC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEA,IAAcY,qBAAqB;AACjC,WAAO,KAAKP,aAAaQ,GAAG,EAAC,IAAK,CAAA;EACpC;EAEUC,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKd,aAAaqB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAK1B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAyB2B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,CAAC,EAAEE,SAAQ,MAAE;AAC9FyB,WAAO,KAAKC,cAAc1B,QAAAA,CAAAA;EAC5B;EAEA,MAAyB2B,YAAYC,UAAiD;AACpF,UAAMjC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUkC,IAAI,YAAY,KAAK/B,iBAAiB;AAChD,WAAO,MAAM,MAAM6B,YAAAA;EACrB;;EAGA,MAAgBpB,cAAc;AAC5B,UAAM,KAAKlC,yBAAyByD,aAAa,YAAA;AAC/C,YAAMnC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIsC,cAAe,MAAMpC,UAAUqC,KAAK;QAAEpB,OAAO;QAAK9B,QAAQ,KAAKJ;MAAmB,CAAA;AACtF,aAAOqD,YAAY1B,SAAS,GAAG;AAC7BD,iBAAS,KAAKjC,aAAakC,SAAS0B,YAAY1B,UAAU,KAAK5B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKiD,cAAcK,WAAAA;AACzBA,sBAAe,MAAMpC,UAAUqC,KAAK;UAAEpB,OAAO;UAAK9B,QAAQ,KAAKJ;QAAmB,CAAA;MACpF;IACF,CAAA;EACF;EAEA,MAAcgD,cAAc1B,UAA4B;AACtD,UAAMf,QAAQ,MAAMgD,eAAeC,UAAUlC,QAAAA;AAC7C,SAAK7B,aAAagE,KAAI,GAAIlD,KAAAA;EAC5B;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadDivinerConfigSchema","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","configSchemas","PayloadDivinerConfigSchema","payloadPairs","_archivistInstance","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","payloadPairsOffset","at","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","indexPayloads","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","hashPairs","push"]}
1
+ {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,8BAEAC,kCAGK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,wBAAN,cASGC,eAAAA;EA/BV,OA+BUA;;;EACR,OAAgBC,gBAAgB;IAACC;;EAEvBC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBX;EACvC;EAEA,IAAca,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBZ;EACrC;EAEUa,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,aAAK/B,eAAe,MAAM0D,eAAe5C,KAAKgB,SAAS0B,YAAYG,GAAG,EAAC,CAAA,CAAA;AACvE7B,iBAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKuD,cAAcJ,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc4D,cAAclC,UAA4B;AACtD,UAAMf,QAAQ,MAAM+C,eAAeG,UAAUnC,QAAAA;AAC7C,SAAK5B,aAAagE,KAAI,GAAInD,KAAAA;EAC5B;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadDivinerConfigSchema","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","configSchemas","PayloadDivinerConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","at","indexPayloads","hashPairs","push"]}
package/dist/node/Diviner.d.cts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/node/Diviner.d.cts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAab,aAAa;CAI5B"}
package/dist/node/Diviner.d.mts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/node/Diviner.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAab,aAAa;CAI5B"}
package/dist/node/Diviner.d.ts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/node/Diviner.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAab,aAAa;CAI5B"}
package/dist/node/index.cjs CHANGED
@@ -42,6 +42,7 @@ var DEFAULT_MAX_INDEX_SIZE = 8e3;
42
42
  var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner_payload_abstract.PayloadDiviner {
43
43
  payloadPairs = [];
44
44
  _archivistInstance;
45
+ _indexOffset;
45
46
  _updatePayloadPairsMutex = new import_async_mutex.Mutex();
46
47
  get indexBatchSize() {
47
48
  return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE;
@@ -49,10 +50,6 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
49
50
  get maxIndexSize() {
50
51
  return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE;
51
52
  }
52
- get payloadPairsOffset() {
53
- var _a;
54
- return (_a = this.payloadPairs.at(-1)) == null ? void 0 : _a[1];
55
- }
56
53
  all(order = "asc", offset) {
57
54
  return order === "asc" ? this.allAsc(offset) : this.allDesc(offset);
58
55
  }
@@ -103,8 +100,8 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
103
100
  throw new Error('Archivist does not support "all"');
104
101
  }
105
102
  }
106
- onArchivistInsert = ({ payloads }) => {
107
- (0, import_forget.forget)(this.indexPayloads(payloads));
103
+ onArchivistInsert = () => {
104
+ (0, import_forget.forget)(this.updateIndex());
108
105
  };
109
106
  async stopHandler(_timeout) {
110
107
  const archivist = await this.archivistInstance(true);
@@ -117,14 +114,15 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
117
114
  const archivist = await this.archivistInstance(true);
118
115
  let newPayloads = await archivist.next({
119
116
  limit: 100,
120
- offset: this.payloadPairsOffset
117
+ offset: this._indexOffset
121
118
  });
122
119
  while (newPayloads.length > 0) {
120
+ this._indexOffset = await import_payload_builder.PayloadBuilder.hash((0, import_assert.assertEx)(newPayloads.at(-1)));
123
121
  (0, import_assert.assertEx)(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
124
122
  await this.indexPayloads(newPayloads);
125
123
  newPayloads = await archivist.next({
126
124
  limit: 100,
127
- offset: this.payloadPairsOffset
125
+ offset: this._indexOffset
128
126
  });
129
127
  }
130
128
  });
package/dist/node/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected get payloadPairsOffset() {\n return this.payloadPairs.at(-1)?.[1]\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {\n forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,yBAAN,MAAMA,+BASHC,+CAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBR;EACvC;EAEA,IAAcU,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBT;EACrC;EAEA,IAAcU,qBAAqB;AA/CrC;AAgDI,YAAO,UAAKP,aAAaQ,GAAG,EAAC,MAAtB,mBAA2B;EACpC;EAEUC,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKd,aAAaqB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAK1B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAyB2B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,+DAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAtG9B;AAuGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,CAAC,EAAEE,SAAQ,MAAE;AAC9FyB,8BAAO,KAAKC,cAAc1B,QAAAA,CAAAA;EAC5B;EAEA,MAAyB2B,YAAYC,UAAiD;AACpF,UAAMjC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUkC,IAAI,YAAY,KAAK/B,iBAAiB;AAChD,WAAO,MAAM,MAAM6B,YAAAA;EACrB;;EAGA,MAAgBpB,cAAc;AAC5B,UAAM,KAAKlC,yBAAyByD,aAAa,YAAA;AAC/C,YAAMnC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIsC,cAAe,MAAMpC,UAAUqC,KAAK;QAAEpB,OAAO;QAAK9B,QAAQ,KAAKJ;MAAmB,CAAA;AACtF,aAAOqD,YAAY1B,SAAS,GAAG;AAC7BD,oCAAS,KAAKjC,aAAakC,SAAS0B,YAAY1B,UAAU,KAAK5B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKiD,cAAcK,WAAAA;AACzBA,sBAAe,MAAMpC,UAAUqC,KAAK;UAAEpB,OAAO;UAAK9B,QAAQ,KAAKJ;QAAmB,CAAA;MACpF;IACF,CAAA;EACF;EAEA,MAAcgD,cAAc1B,UAA4B;AACtD,UAAMf,QAAQ,MAAMgD,sCAAeC,UAAUlC,QAAAA;AAC7C,SAAK7B,aAAagE,KAAI,GAAIlD,KAAAA;EAC5B;AACF;AAjHUf;AACR,cAVWD,wBAUKmE,iBAAgB;EAACC;;AAV5B,IAAMpE,wBAAN;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","payloadPairsOffset","at","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","indexPayloads","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}
1
+ {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,yBAAN,MAAMA,+BASHC,+CAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBT;EACvC;EAEA,IAAcW,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBV;EACrC;EAEUW,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,+DAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAnG9B;AAoGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,aAAK/B,eAAe,MAAM0D,sCAAe5C,SAAKgB,wBAAS0B,YAAYG,GAAG,EAAC,CAAA,CAAA;AACvE7B,oCAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKuD,cAAcJ,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc4D,cAAclC,UAA4B;AACtD,UAAMf,QAAQ,MAAM+C,sCAAeG,UAAUnC,QAAAA;AAC7C,SAAK5B,aAAagE,KAAI,GAAInD,KAAAA;EAC5B;AACF;AAhHUd;AACR,cAVWD,wBAUKmE,iBAAgB;EAACC;;AAV5B,IAAMpE,wBAAN;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","at","indexPayloads","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}
package/dist/node/index.js CHANGED
@@ -18,6 +18,7 @@ var DEFAULT_MAX_INDEX_SIZE = 8e3;
18
18
  var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner {
19
19
  payloadPairs = [];
20
20
  _archivistInstance;
21
+ _indexOffset;
21
22
  _updatePayloadPairsMutex = new Mutex();
22
23
  get indexBatchSize() {
23
24
  return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE;
@@ -25,10 +26,6 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
25
26
  get maxIndexSize() {
26
27
  return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE;
27
28
  }
28
- get payloadPairsOffset() {
29
- var _a;
30
- return (_a = this.payloadPairs.at(-1)) == null ? void 0 : _a[1];
31
- }
32
29
  all(order = "asc", offset) {
33
30
  return order === "asc" ? this.allAsc(offset) : this.allDesc(offset);
34
31
  }
@@ -79,8 +76,8 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
79
76
  throw new Error('Archivist does not support "all"');
80
77
  }
81
78
  }
82
- onArchivistInsert = ({ payloads }) => {
83
- forget(this.indexPayloads(payloads));
79
+ onArchivistInsert = () => {
80
+ forget(this.updateIndex());
84
81
  };
85
82
  async stopHandler(_timeout) {
86
83
  const archivist = await this.archivistInstance(true);
@@ -93,14 +90,15 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
93
90
  const archivist = await this.archivistInstance(true);
94
91
  let newPayloads = await archivist.next({
95
92
  limit: 100,
96
- offset: this.payloadPairsOffset
93
+ offset: this._indexOffset
97
94
  });
98
95
  while (newPayloads.length > 0) {
96
+ this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)));
99
97
  assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
100
98
  await this.indexPayloads(newPayloads);
101
99
  newPayloads = await archivist.next({
102
100
  limit: 100,
103
- offset: this.payloadPairsOffset
101
+ offset: this._indexOffset
104
102
  });
105
103
  }
106
104
  });
package/dist/node/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected get payloadPairsOffset() {\n return this.payloadPairs.at(-1)?.[1]\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {\n forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,8BAEAC,kCAGK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,yBAAN,MAAMA,+BASHC,eAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBR;EACvC;EAEA,IAAcU,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBT;EACrC;EAEA,IAAcU,qBAAqB;AA/CrC;AAgDI,YAAO,UAAKP,aAAaQ,GAAG,EAAC,MAAtB,mBAA2B;EACpC;EAEUC,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKd,aAAaqB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAK1B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAyB2B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,kCAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAtG9B;AAuGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,CAAC,EAAEE,SAAQ,MAAE;AAC9FyB,WAAO,KAAKC,cAAc1B,QAAAA,CAAAA;EAC5B;EAEA,MAAyB2B,YAAYC,UAAiD;AACpF,UAAMjC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUkC,IAAI,YAAY,KAAK/B,iBAAiB;AAChD,WAAO,MAAM,MAAM6B,YAAAA;EACrB;;EAGA,MAAgBpB,cAAc;AAC5B,UAAM,KAAKlC,yBAAyByD,aAAa,YAAA;AAC/C,YAAMnC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIsC,cAAe,MAAMpC,UAAUqC,KAAK;QAAEpB,OAAO;QAAK9B,QAAQ,KAAKJ;MAAmB,CAAA;AACtF,aAAOqD,YAAY1B,SAAS,GAAG;AAC7BD,iBAAS,KAAKjC,aAAakC,SAAS0B,YAAY1B,UAAU,KAAK5B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKiD,cAAcK,WAAAA;AACzBA,sBAAe,MAAMpC,UAAUqC,KAAK;UAAEpB,OAAO;UAAK9B,QAAQ,KAAKJ;QAAmB,CAAA;MACpF;IACF,CAAA;EACF;EAEA,MAAcgD,cAAc1B,UAA4B;AACtD,UAAMf,QAAQ,MAAMgD,eAAeC,UAAUlC,QAAAA;AAC7C,SAAK7B,aAAagE,KAAI,GAAIlD,KAAAA;EAC5B;AACF;AAjHUf;AACR,cAVWD,wBAUKmE,iBAAgB;EAACC;;AAV5B,IAAMpE,wBAAN;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadDivinerConfigSchema","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","payloadPairsOffset","at","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","indexPayloads","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}
1
+ {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,8BAEAC,kCAGK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,yBAAN,MAAMA,+BASHC,eAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBT;EACvC;EAEA,IAAcW,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBV;EACrC;EAEUW,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,kCAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAnG9B;AAoGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B,aAAK/B,eAAe,MAAM0D,eAAe5C,KAAKgB,SAAS0B,YAAYG,GAAG,EAAC,CAAA,CAAA;AACvE7B,iBAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKuD,cAAcJ,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc4D,cAAclC,UAA4B;AACtD,UAAMf,QAAQ,MAAM+C,eAAeG,UAAUnC,QAAAA;AAC7C,SAAK5B,aAAagE,KAAI,GAAInD,KAAAA;EAC5B;AACF;AAhHUd;AACR,cAVWD,wBAUKmE,iBAAgB;EAACC;;AAV5B,IAAMpE,wBAAN;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadDivinerConfigSchema","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","at","indexPayloads","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}
package/package.json CHANGED
@@ -10,26 +10,26 @@
10
10
  "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js/issues"
11
11
  },
12
12
  "dependencies": {
13
- "@xylabs/assert": "^3.1.7",
14
- "@xylabs/forget": "^3.1.7",
15
- "@xylabs/hex": "^3.1.7",
16
- "@xylabs/object": "^3.1.7",
17
- "@xyo-network/archivist-model": "~2.95.6",
18
- "@xyo-network/diviner-model": "~2.95.6",
19
- "@xyo-network/diviner-payload-abstract": "~2.95.6",
20
- "@xyo-network/diviner-payload-model": "~2.95.6",
21
- "@xyo-network/module-events": "~2.95.6",
22
- "@xyo-network/payload-builder": "~2.95.6",
23
- "@xyo-network/payload-model": "~2.95.6",
13
+ "@xylabs/assert": "^3.1.8",
14
+ "@xylabs/forget": "^3.1.8",
15
+ "@xylabs/hex": "^3.1.8",
16
+ "@xylabs/object": "^3.1.8",
17
+ "@xyo-network/archivist-model": "~2.95.8",
18
+ "@xyo-network/diviner-model": "~2.95.8",
19
+ "@xyo-network/diviner-payload-abstract": "~2.95.8",
20
+ "@xyo-network/diviner-payload-model": "~2.95.8",
21
+ "@xyo-network/module-events": "~2.95.8",
22
+ "@xyo-network/payload-builder": "~2.95.8",
23
+ "@xyo-network/payload-model": "~2.95.8",
24
24
  "async-mutex": "^0.5.0"
25
25
  },
26
26
  "devDependencies": {
27
- "@xylabs/ts-scripts-yarn3": "^3.9.0",
28
- "@xylabs/tsconfig": "^3.9.0",
29
- "@xyo-network/account": "~2.95.6",
30
- "@xyo-network/archivist-memory": "~2.95.6",
31
- "@xyo-network/node-memory": "~2.95.6",
32
- "@xyo-network/payload-builder": "~2.95.6",
27
+ "@xylabs/ts-scripts-yarn3": "^3.9.1",
28
+ "@xylabs/tsconfig": "^3.9.1",
29
+ "@xyo-network/account": "~2.95.8",
30
+ "@xyo-network/archivist-memory": "~2.95.8",
31
+ "@xyo-network/node-memory": "~2.95.8",
32
+ "@xyo-network/payload-builder": "~2.95.8",
33
33
  "typescript": "^5.4.5"
34
34
  },
35
35
  "description": "Primary SDK for using XYO Protocol 2.0",
@@ -71,6 +71,6 @@
71
71
  "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
72
72
  },
73
73
  "sideEffects": false,
74
- "version": "2.95.6",
74
+ "version": "2.95.8",
75
75
  "type": "module"
76
76
  }
package/src/Diviner.ts CHANGED
@@ -35,6 +35,7 @@ export class GenericPayloadDiviner<
35
35
  protected payloadPairs: [WithMeta<TOut>, Hash][] = []
36
36
 
37
37
  private _archivistInstance?: ArchivistInstance
38
+ private _indexOffset?: Hash
38
39
  private _updatePayloadPairsMutex = new Mutex()
39
40
 
40
41
  protected get indexBatchSize() {
@@ -45,10 +46,6 @@ export class GenericPayloadDiviner<
45
46
  return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE
46
47
  }
47
48
 
48
- protected get payloadPairsOffset() {
49
- return this.payloadPairs.at(-1)?.[1]
50
- }
51
-
52
49
  protected all(order: Order = 'asc', offset?: Hash) {
53
50
  return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)
54
51
  }
@@ -115,8 +112,9 @@ export class GenericPayloadDiviner<
115
112
  }
116
113
  }
117
114
 
118
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {
119
- forget(this.indexPayloads(payloads as WithMeta<TOut>[]))
115
+ protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {
116
+ //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))
117
+ forget(this.updateIndex())
120
118
  }
121
119
 
122
120
  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {
@@ -129,11 +127,12 @@ export class GenericPayloadDiviner<
129
127
  protected async updateIndex() {
130
128
  await this._updatePayloadPairsMutex.runExclusive(async () => {
131
129
  const archivist = await this.archivistInstance(true)
132
- let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]
130
+ let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]
133
131
  while (newPayloads.length > 0) {
132
+ this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))
134
133
  assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')
135
134
  await this.indexPayloads(newPayloads)
136
- newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]
135
+ newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]
137
136
  }
138
137
  })
139
138
  }