@xyo-network/diviner-payload-generic 2.95.7 → 2.95.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. package/dist/browser/Diviner.d.cts +1 -1
  2. package/dist/browser/Diviner.d.cts.map +1 -1
  3. package/dist/browser/Diviner.d.mts +1 -1
  4. package/dist/browser/Diviner.d.mts.map +1 -1
  5. package/dist/browser/Diviner.d.ts +1 -1
  6. package/dist/browser/Diviner.d.ts.map +1 -1
  7. package/dist/browser/index.cjs +10 -9
  8. package/dist/browser/index.cjs.map +1 -1
  9. package/dist/browser/index.js +10 -9
  10. package/dist/browser/index.js.map +1 -1
  11. package/dist/node/Diviner.d.cts +1 -1
  12. package/dist/node/Diviner.d.cts.map +1 -1
  13. package/dist/node/Diviner.d.mts +1 -1
  14. package/dist/node/Diviner.d.mts.map +1 -1
  15. package/dist/node/Diviner.d.ts +1 -1
  16. package/dist/node/Diviner.d.ts.map +1 -1
  17. package/dist/node/index.cjs +10 -10
  18. package/dist/node/index.cjs.map +1 -1
  19. package/dist/node/index.js +10 -10
  20. package/dist/node/index.js.map +1 -1
  21. package/package.json +12 -12
  22. package/src/Diviner.ts +12 -11
package/dist/browser/Diviner.d.cts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/browser/Diviner.d.cts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
package/dist/browser/Diviner.d.mts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/browser/Diviner.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
package/dist/browser/Diviner.d.ts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/browser/Diviner.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
package/dist/browser/index.cjs CHANGED
@@ -43,6 +43,7 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
43
43
  ];
44
44
  payloadPairs = [];
45
45
  _archivistInstance;
46
+ _indexOffset;
46
47
  _updatePayloadPairsMutex = new import_async_mutex.Mutex();
47
48
  get indexBatchSize() {
48
49
  return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE;
@@ -50,20 +51,17 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
50
51
  get maxIndexSize() {
51
52
  return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE;
52
53
  }
53
- get payloadPairsOffset() {
54
- return this.payloadPairs.at(-1)?.[1];
55
- }
56
54
  all(order = "asc", offset) {
57
55
  return order === "asc" ? this.allAsc(offset) : this.allDesc(offset);
58
56
  }
59
57
  allAsc(offset) {
60
58
  const pairs = this.payloadPairs;
61
- const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
59
+ const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1;
62
60
  return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
63
61
  }
64
62
  allDesc(offset) {
65
63
  const pairs = this.payloadPairs.reverse();
66
- const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
64
+ const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1;
67
65
  return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
68
66
  }
69
67
  async archivistInstance(required = false) {
@@ -102,8 +100,8 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
102
100
  throw new Error('Archivist does not support "all"');
103
101
  }
104
102
  }
105
- onArchivistInsert = ({ payloads }) => {
106
- (0, import_forget.forget)(this.indexPayloads(payloads));
103
+ onArchivistInsert = () => {
104
+ (0, import_forget.forget)(this.updateIndex());
107
105
  };
108
106
  async stopHandler(_timeout) {
109
107
  const archivist = await this.archivistInstance(true);
@@ -116,14 +114,16 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
116
114
  const archivist = await this.archivistInstance(true);
117
115
  let newPayloads = await archivist.next({
118
116
  limit: 100,
119
- offset: this.payloadPairsOffset
117
+ offset: this._indexOffset
120
118
  });
121
119
  while (newPayloads.length > 0) {
120
+ console.log("newPayloads", newPayloads);
121
+ this._indexOffset = await import_payload_builder.PayloadBuilder.hash((0, import_assert.assertEx)(newPayloads.at(-1)));
122
122
  (0, import_assert.assertEx)(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
123
123
  await this.indexPayloads(newPayloads);
124
124
  newPayloads = await archivist.next({
125
125
  limit: 100,
126
- offset: this.payloadPairsOffset
126
+ offset: this._indexOffset
127
127
  });
128
128
  }
129
129
  });
@@ -131,6 +131,7 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
131
131
  async indexPayloads(payloads) {
132
132
  const pairs = await import_payload_builder.PayloadBuilder.hashPairs(payloads);
133
133
  this.payloadPairs.push(...pairs);
134
+ return (0, import_assert.assertEx)(pairs.at(-1))[1];
134
135
  }
135
136
  };
136
137
  //# sourceMappingURL=index.cjs.map
package/dist/browser/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected get payloadPairsOffset() {\n return this.payloadPairs.at(-1)?.[1]\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {\n forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,wBAAN,cASGC,+CAAAA;EA/BV,OA+BUA;;;EACR,OAAgBC,gBAAgB;IAACC;;EAEvBC,eAAyC,CAAA;EAE3CC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEA,IAAcY,qBAAqB;AACjC,WAAO,KAAKP,aAAaQ,GAAG,EAAC,IAAK,CAAA;EACpC;EAEUC,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKd,aAAaqB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAK1B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAyB2B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,CAAC,EAAEE,SAAQ,MAAE;AAC9FyB,8BAAO,KAAKC,cAAc1B,QAAAA,CAAAA;EAC5B;EAEA,MAAyB2B,YAAYC,UAAiD;AACpF,UAAMjC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUkC,IAAI,YAAY,KAAK/B,iBAAiB;AAChD,WAAO,MAAM,MAAM6B,YAAAA;EACrB;;EAGA,MAAgBpB,cAAc;AAC5B,UAAM,KAAKlC,yBAAyByD,aAAa,YAAA;AAC/C,YAAMnC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIsC,cAAe,MAAMpC,UAAUqC,KAAK;QAAEpB,OAAO;QAAK9B,QAAQ,KAAKJ;MAAmB,CAAA;AACtF,aAAOqD,YAAY1B,SAAS,GAAG;AAC7BD,oCAAS,KAAKjC,aAAakC,SAAS0B,YAAY1B,UAAU,KAAK5B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKiD,cAAcK,WAAAA;AACzBA,sBAAe,MAAMpC,UAAUqC,KAAK;UAAEpB,OAAO;UAAK9B,QAAQ,KAAKJ;QAAmB,CAAA;MACpF;IACF,CAAA;EACF;EAEA,MAAcgD,cAAc1B,UAA4B;AACtD,UAAMf,QAAQ,MAAMgD,sCAAeC,UAAUlC,QAAAA;AAC7C,SAAK7B,aAAagE,KAAI,GAAIlD,KAAAA;EAC5B;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","configSchemas","PayloadDivinerConfigSchema","payloadPairs","_archivistInstance","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","payloadPairsOffset","at","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","indexPayloads","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","hashPairs","push"]}
1
+ {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n console.log('newPayloads', newPayloads)\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,wBAAN,cASGC,+CAAAA;EA/BV,OA+BUA;;;EACR,OAAgBC,gBAAgB;IAACC;;EAEvBC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBX;EACvC;EAEA,IAAca,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBZ;EACrC;EAEUa,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B2B,gBAAQC,IAAI,eAAeH,WAAAA;AAC3B,aAAKxD,eAAe,MAAM4D,sCAAe9C,SAAKgB,wBAAS0B,YAAYK,GAAG,EAAC,CAAA,CAAA;AACvE/B,oCAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKyD,cAAcN,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc8D,cAAcpC,UAA2C;AACrE,UAAMf,QAAQ,MAAMiD,sCAAeG,UAAUrC,QAAAA;AAC7C,SAAK5B,aAAakE,KAAI,GAAIrD,KAAAA;AAC1B,eAAOmB,wBAASnB,MAAMkD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","configSchemas","PayloadDivinerConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","console","log","PayloadBuilder","at","indexPayloads","hashPairs","push"]}
package/dist/browser/index.js CHANGED
@@ -19,6 +19,7 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
19
19
  ];
20
20
  payloadPairs = [];
21
21
  _archivistInstance;
22
+ _indexOffset;
22
23
  _updatePayloadPairsMutex = new Mutex();
23
24
  get indexBatchSize() {
24
25
  return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE;
@@ -26,20 +27,17 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
26
27
  get maxIndexSize() {
27
28
  return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE;
28
29
  }
29
- get payloadPairsOffset() {
30
- return this.payloadPairs.at(-1)?.[1];
31
- }
32
30
  all(order = "asc", offset) {
33
31
  return order === "asc" ? this.allAsc(offset) : this.allDesc(offset);
34
32
  }
35
33
  allAsc(offset) {
36
34
  const pairs = this.payloadPairs;
37
- const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
35
+ const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1;
38
36
  return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
39
37
  }
40
38
  allDesc(offset) {
41
39
  const pairs = this.payloadPairs.reverse();
42
- const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
40
+ const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1;
43
41
  return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
44
42
  }
45
43
  async archivistInstance(required = false) {
@@ -78,8 +76,8 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
78
76
  throw new Error('Archivist does not support "all"');
79
77
  }
80
78
  }
81
- onArchivistInsert = ({ payloads }) => {
82
- forget(this.indexPayloads(payloads));
79
+ onArchivistInsert = () => {
80
+ forget(this.updateIndex());
83
81
  };
84
82
  async stopHandler(_timeout) {
85
83
  const archivist = await this.archivistInstance(true);
@@ -92,14 +90,16 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
92
90
  const archivist = await this.archivistInstance(true);
93
91
  let newPayloads = await archivist.next({
94
92
  limit: 100,
95
- offset: this.payloadPairsOffset
93
+ offset: this._indexOffset
96
94
  });
97
95
  while (newPayloads.length > 0) {
96
+ console.log("newPayloads", newPayloads);
97
+ this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)));
98
98
  assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
99
99
  await this.indexPayloads(newPayloads);
100
100
  newPayloads = await archivist.next({
101
101
  limit: 100,
102
- offset: this.payloadPairsOffset
102
+ offset: this._indexOffset
103
103
  });
104
104
  }
105
105
  });
@@ -107,6 +107,7 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
107
107
  async indexPayloads(payloads) {
108
108
  const pairs = await PayloadBuilder.hashPairs(payloads);
109
109
  this.payloadPairs.push(...pairs);
110
+ return assertEx(pairs.at(-1))[1];
110
111
  }
111
112
  };
112
113
  export {
package/dist/browser/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected get payloadPairsOffset() {\n return this.payloadPairs.at(-1)?.[1]\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {\n forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,8BAEAC,kCAGK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,wBAAN,cASGC,eAAAA;EA/BV,OA+BUA;;;EACR,OAAgBC,gBAAgB;IAACC;;EAEvBC,eAAyC,CAAA;EAE3CC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBV;EACvC;EAEA,IAAcY,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBX;EACrC;EAEA,IAAcY,qBAAqB;AACjC,WAAO,KAAKP,aAAaQ,GAAG,EAAC,IAAK,CAAA;EACpC;EAEUC,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKd,aAAaqB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAK1B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAyB2B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,CAAC,EAAEE,SAAQ,MAAE;AAC9FyB,WAAO,KAAKC,cAAc1B,QAAAA,CAAAA;EAC5B;EAEA,MAAyB2B,YAAYC,UAAiD;AACpF,UAAMjC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUkC,IAAI,YAAY,KAAK/B,iBAAiB;AAChD,WAAO,MAAM,MAAM6B,YAAAA;EACrB;;EAGA,MAAgBpB,cAAc;AAC5B,UAAM,KAAKlC,yBAAyByD,aAAa,YAAA;AAC/C,YAAMnC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIsC,cAAe,MAAMpC,UAAUqC,KAAK;QAAEpB,OAAO;QAAK9B,QAAQ,KAAKJ;MAAmB,CAAA;AACtF,aAAOqD,YAAY1B,SAAS,GAAG;AAC7BD,iBAAS,KAAKjC,aAAakC,SAAS0B,YAAY1B,UAAU,KAAK5B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKiD,cAAcK,WAAAA;AACzBA,sBAAe,MAAMpC,UAAUqC,KAAK;UAAEpB,OAAO;UAAK9B,QAAQ,KAAKJ;QAAmB,CAAA;MACpF;IACF,CAAA;EACF;EAEA,MAAcgD,cAAc1B,UAA4B;AACtD,UAAMf,QAAQ,MAAMgD,eAAeC,UAAUlC,QAAAA;AAC7C,SAAK7B,aAAagE,KAAI,GAAIlD,KAAAA;EAC5B;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadDivinerConfigSchema","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","configSchemas","PayloadDivinerConfigSchema","payloadPairs","_archivistInstance","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","payloadPairsOffset","at","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","indexPayloads","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","hashPairs","push"]}
1
+ {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n console.log('newPayloads', newPayloads)\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,8BAEAC,kCAGK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,wBAAN,cASGC,eAAAA;EA/BV,OA+BUA;;;EACR,OAAgBC,gBAAgB;IAACC;;EAEvBC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBX;EACvC;EAEA,IAAca,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBZ;EACrC;EAEUa,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,iBAAiB;AAChD,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,SAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAO5B,UAAU6B,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,YAAYA,UAAU6B,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B2B,gBAAQC,IAAI,eAAeH,WAAAA;AAC3B,aAAKxD,eAAe,MAAM4D,eAAe9C,KAAKgB,SAAS0B,YAAYK,GAAG,EAAC,CAAA,CAAA;AACvE/B,iBAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKyD,cAAcN,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc8D,cAAcpC,UAA2C;AACrE,UAAMf,QAAQ,MAAMiD,eAAeG,UAAUrC,QAAAA;AAC7C,SAAK5B,aAAakE,KAAI,GAAIrD,KAAAA;AAC1B,WAAOmB,SAASnB,MAAMkD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadDivinerConfigSchema","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","configSchemas","PayloadDivinerConfigSchema","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","console","log","PayloadBuilder","at","indexPayloads","hashPairs","push"]}
package/dist/node/Diviner.d.cts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/node/Diviner.d.cts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
package/dist/node/Diviner.d.mts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/node/Diviner.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
package/dist/node/Diviner.d.ts CHANGED
@@ -9,10 +9,10 @@ export declare class GenericPayloadDiviner<TParams extends PayloadDivinerParams
9
9
  static configSchemas: "network.xyo.diviner.payload.config"[];
10
10
  protected payloadPairs: [WithMeta<TOut>, Hash][];
11
11
  private _archivistInstance?;
12
+ private _indexOffset?;
12
13
  private _updatePayloadPairsMutex;
13
14
  protected get indexBatchSize(): number;
14
15
  protected get maxIndexSize(): number;
15
- protected get payloadPairsOffset(): Lowercase<string> | undefined;
16
16
  protected all(order?: Order, offset?: Hash): WithMeta<TOut>[];
17
17
  protected allAsc(offset?: Hash): WithMeta<TOut>[];
18
18
  protected allDesc(offset?: Hash): WithMeta<TOut>[];
package/dist/node/Diviner.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,KAAK,kBAAkB,kCAE/B;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAE/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAYb,aAAa;CAI5B"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAC1F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAEL,KAAK,EAEL,oBAAoB,EACpB,0BAA0B,EAC3B,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE1D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAM9D,qBAAa,qBAAqB,CAChC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,GAAG,SAAS,0BAA0B,GAAG,0BAA0B,EACnE,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACtD,OAAgB,aAAa,yCAA+B;IAE5D,SAAS,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAK;IAErD,OAAO,CAAC,kBAAkB,CAAC,CAAmB;IAC9C,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,wBAAwB,CAAc;IAE9C,SAAS,KAAK,cAAc,WAE3B;IAED,SAAS,KAAK,YAAY,WAEzB;IAED,SAAS,CAAC,GAAG,CAAC,KAAK,GAAE,KAAa,EAAE,MAAM,CAAC,EAAE,IAAI;IAIjD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI;IAM9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;cAMN,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;cAC3D,iBAAiB,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;cAa7D,aAAa,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAoCnF,SAAS,CAAC,iBAAiB,EAAE,aAAa,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,CAG/E;cAEwB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;cAOrE,WAAW;YAcb,aAAa;CAK5B"}
package/dist/node/index.cjs CHANGED
@@ -42,6 +42,7 @@ var DEFAULT_MAX_INDEX_SIZE = 8e3;
42
42
  var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner_payload_abstract.PayloadDiviner {
43
43
  payloadPairs = [];
44
44
  _archivistInstance;
45
+ _indexOffset;
45
46
  _updatePayloadPairsMutex = new import_async_mutex.Mutex();
46
47
  get indexBatchSize() {
47
48
  return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE;
@@ -49,21 +50,17 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
49
50
  get maxIndexSize() {
50
51
  return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE;
51
52
  }
52
- get payloadPairsOffset() {
53
- var _a;
54
- return (_a = this.payloadPairs.at(-1)) == null ? void 0 : _a[1];
55
- }
56
53
  all(order = "asc", offset) {
57
54
  return order === "asc" ? this.allAsc(offset) : this.allDesc(offset);
58
55
  }
59
56
  allAsc(offset) {
60
57
  const pairs = this.payloadPairs;
61
- const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
58
+ const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1;
62
59
  return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
63
60
  }
64
61
  allDesc(offset) {
65
62
  const pairs = this.payloadPairs.reverse();
66
- const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
63
+ const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1;
67
64
  return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
68
65
  }
69
66
  async archivistInstance(required = false) {
@@ -103,8 +100,8 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
103
100
  throw new Error('Archivist does not support "all"');
104
101
  }
105
102
  }
106
- onArchivistInsert = ({ payloads }) => {
107
- (0, import_forget.forget)(this.indexPayloads(payloads));
103
+ onArchivistInsert = () => {
104
+ (0, import_forget.forget)(this.updateIndex());
108
105
  };
109
106
  async stopHandler(_timeout) {
110
107
  const archivist = await this.archivistInstance(true);
@@ -117,14 +114,16 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
117
114
  const archivist = await this.archivistInstance(true);
118
115
  let newPayloads = await archivist.next({
119
116
  limit: 100,
120
- offset: this.payloadPairsOffset
117
+ offset: this._indexOffset
121
118
  });
122
119
  while (newPayloads.length > 0) {
120
+ console.log("newPayloads", newPayloads);
121
+ this._indexOffset = await import_payload_builder.PayloadBuilder.hash((0, import_assert.assertEx)(newPayloads.at(-1)));
123
122
  (0, import_assert.assertEx)(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
124
123
  await this.indexPayloads(newPayloads);
125
124
  newPayloads = await archivist.next({
126
125
  limit: 100,
127
- offset: this.payloadPairsOffset
126
+ offset: this._indexOffset
128
127
  });
129
128
  }
130
129
  });
@@ -132,6 +131,7 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
132
131
  async indexPayloads(payloads) {
133
132
  const pairs = await import_payload_builder.PayloadBuilder.hashPairs(payloads);
134
133
  this.payloadPairs.push(...pairs);
134
+ return (0, import_assert.assertEx)(pairs.at(-1))[1];
135
135
  }
136
136
  };
137
137
  __name(_GenericPayloadDiviner, "GenericPayloadDiviner");
package/dist/node/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected get payloadPairsOffset() {\n return this.payloadPairs.at(-1)?.[1]\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {\n forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,yBAAN,MAAMA,+BASHC,+CAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBR;EACvC;EAEA,IAAcU,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBT;EACrC;EAEA,IAAcU,qBAAqB;AA/CrC;AAgDI,YAAO,UAAKP,aAAaQ,GAAG,EAAC,MAAtB,mBAA2B;EACpC;EAEUC,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKd,aAAaqB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAK1B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAyB2B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,+DAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAtG9B;AAuGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,CAAC,EAAEE,SAAQ,MAAE;AAC9FyB,8BAAO,KAAKC,cAAc1B,QAAAA,CAAAA;EAC5B;EAEA,MAAyB2B,YAAYC,UAAiD;AACpF,UAAMjC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUkC,IAAI,YAAY,KAAK/B,iBAAiB;AAChD,WAAO,MAAM,MAAM6B,YAAAA;EACrB;;EAGA,MAAgBpB,cAAc;AAC5B,UAAM,KAAKlC,yBAAyByD,aAAa,YAAA;AAC/C,YAAMnC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIsC,cAAe,MAAMpC,UAAUqC,KAAK;QAAEpB,OAAO;QAAK9B,QAAQ,KAAKJ;MAAmB,CAAA;AACtF,aAAOqD,YAAY1B,SAAS,GAAG;AAC7BD,oCAAS,KAAKjC,aAAakC,SAAS0B,YAAY1B,UAAU,KAAK5B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKiD,cAAcK,WAAAA;AACzBA,sBAAe,MAAMpC,UAAUqC,KAAK;UAAEpB,OAAO;UAAK9B,QAAQ,KAAKJ;QAAmB,CAAA;MACpF;IACF,CAAA;EACF;EAEA,MAAcgD,cAAc1B,UAA4B;AACtD,UAAMf,QAAQ,MAAMgD,sCAAeC,UAAUlC,QAAAA;AAC7C,SAAK7B,aAAagE,KAAI,GAAIlD,KAAAA;EAC5B;AACF;AAjHUf;AACR,cAVWD,wBAUKmE,iBAAgB;EAACC;;AAV5B,IAAMpE,wBAAN;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","payloadPairsOffset","at","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","indexPayloads","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}
1
+ {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n console.log('newPayloads', newPayloads)\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,yBAAN,MAAMA,+BASHC,+CAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBT;EACvC;EAEA,IAAcW,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBV;EACrC;EAEUW,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,+DAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAnG9B;AAoGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B2B,gBAAQC,IAAI,eAAeH,WAAAA;AAC3B,aAAKxD,eAAe,MAAM4D,sCAAe9C,SAAKgB,wBAAS0B,YAAYK,GAAG,EAAC,CAAA,CAAA;AACvE/B,oCAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKyD,cAAcN,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc8D,cAAcpC,UAA2C;AACrE,UAAMf,QAAQ,MAAMiD,sCAAeG,UAAUrC,QAAAA;AAC7C,SAAK5B,aAAakE,KAAI,GAAIrD,KAAAA;AAC1B,eAAOmB,wBAASnB,MAAMkD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAlHUhE;AACR,cAVWD,wBAUKqE,iBAAgB;EAACC;;AAV5B,IAAMtE,wBAAN;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","console","log","PayloadBuilder","at","indexPayloads","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}
package/dist/node/index.js CHANGED
@@ -18,6 +18,7 @@ var DEFAULT_MAX_INDEX_SIZE = 8e3;
18
18
  var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner {
19
19
  payloadPairs = [];
20
20
  _archivistInstance;
21
+ _indexOffset;
21
22
  _updatePayloadPairsMutex = new Mutex();
22
23
  get indexBatchSize() {
23
24
  return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE;
@@ -25,21 +26,17 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
25
26
  get maxIndexSize() {
26
27
  return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE;
27
28
  }
28
- get payloadPairsOffset() {
29
- var _a;
30
- return (_a = this.payloadPairs.at(-1)) == null ? void 0 : _a[1];
31
- }
32
29
  all(order = "asc", offset) {
33
30
  return order === "asc" ? this.allAsc(offset) : this.allDesc(offset);
34
31
  }
35
32
  allAsc(offset) {
36
33
  const pairs = this.payloadPairs;
37
- const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
34
+ const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1;
38
35
  return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
39
36
  }
40
37
  allDesc(offset) {
41
38
  const pairs = this.payloadPairs.reverse();
42
- const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0;
39
+ const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1;
43
40
  return this.payloadPairs.slice(startIndex).map(([payload]) => payload);
44
41
  }
45
42
  async archivistInstance(required = false) {
@@ -79,8 +76,8 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
79
76
  throw new Error('Archivist does not support "all"');
80
77
  }
81
78
  }
82
- onArchivistInsert = ({ payloads }) => {
83
- forget(this.indexPayloads(payloads));
79
+ onArchivistInsert = () => {
80
+ forget(this.updateIndex());
84
81
  };
85
82
  async stopHandler(_timeout) {
86
83
  const archivist = await this.archivistInstance(true);
@@ -93,14 +90,16 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
93
90
  const archivist = await this.archivistInstance(true);
94
91
  let newPayloads = await archivist.next({
95
92
  limit: 100,
96
- offset: this.payloadPairsOffset
93
+ offset: this._indexOffset
97
94
  });
98
95
  while (newPayloads.length > 0) {
96
+ console.log("newPayloads", newPayloads);
97
+ this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)));
99
98
  assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => "maxIndexSize exceeded");
100
99
  await this.indexPayloads(newPayloads);
101
100
  newPayloads = await archivist.next({
102
101
  limit: 100,
103
- offset: this.payloadPairsOffset
102
+ offset: this._indexOffset
104
103
  });
105
104
  }
106
105
  });
@@ -108,6 +107,7 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
108
107
  async indexPayloads(payloads) {
109
108
  const pairs = await PayloadBuilder.hashPairs(payloads);
110
109
  this.payloadPairs.push(...pairs);
110
+ return assertEx(pairs.at(-1))[1];
111
111
  }
112
112
  };
113
113
  __name(_GenericPayloadDiviner, "GenericPayloadDiviner");
package/dist/node/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected get payloadPairsOffset() {\n return this.payloadPairs.at(-1)?.[1]\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {\n forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]) {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n }\n}\n"],"mappings":";;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,8BAEAC,kCAGK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,yBAAN,MAAMA,+BASHC,eAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBR;EACvC;EAEA,IAAcU,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBT;EACrC;EAEA,IAAcU,qBAAqB;AA/CrC;AAgDI,YAAO,UAAKP,aAAaQ,GAAG,EAAC,MAAtB,mBAA2B;EACpC;EAEUC,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKd,aAAaqB,QAAO;AACvC,UAAMN,aAAaJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,IAAI;AAClF,WAAO,KAAKX,aAAakB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAK1B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAyB2B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,kCAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAtG9B;AAuGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,CAAC,EAAEE,SAAQ,MAAE;AAC9FyB,WAAO,KAAKC,cAAc1B,QAAAA,CAAAA;EAC5B;EAEA,MAAyB2B,YAAYC,UAAiD;AACpF,UAAMjC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUkC,IAAI,YAAY,KAAK/B,iBAAiB;AAChD,WAAO,MAAM,MAAM6B,YAAAA;EACrB;;EAGA,MAAgBpB,cAAc;AAC5B,UAAM,KAAKlC,yBAAyByD,aAAa,YAAA;AAC/C,YAAMnC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIsC,cAAe,MAAMpC,UAAUqC,KAAK;QAAEpB,OAAO;QAAK9B,QAAQ,KAAKJ;MAAmB,CAAA;AACtF,aAAOqD,YAAY1B,SAAS,GAAG;AAC7BD,iBAAS,KAAKjC,aAAakC,SAAS0B,YAAY1B,UAAU,KAAK5B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKiD,cAAcK,WAAAA;AACzBA,sBAAe,MAAMpC,UAAUqC,KAAK;UAAEpB,OAAO;UAAK9B,QAAQ,KAAKJ;QAAmB,CAAA;MACpF;IACF,CAAA;EACF;EAEA,MAAcgD,cAAc1B,UAA4B;AACtD,UAAMf,QAAQ,MAAMgD,eAAeC,UAAUlC,QAAAA;AAC7C,SAAK7B,aAAagE,KAAI,GAAIlD,KAAAA;EAC5B;AACF;AAjHUf;AACR,cAVWD,wBAUKmE,iBAAgB;EAACC;;AAV5B,IAAMpE,wBAAN;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadDivinerConfigSchema","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","payloadPairsOffset","at","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","indexPayloads","stopHandler","_timeout","off","runExclusive","newPayloads","next","PayloadBuilder","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}
1
+ {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n isPayloadDivinerQueryPayload,\n Order,\n PayloadDivinerConfigSchema,\n PayloadDivinerParams,\n PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport class GenericPayloadDiviner<\n TParams extends PayloadDivinerParams = PayloadDivinerParams,\n TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n static override configSchemas = [PayloadDivinerConfigSchema]\n\n protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n private _archivistInstance?: ArchivistInstance\n private _indexOffset?: Hash\n private _updatePayloadPairsMutex = new Mutex()\n\n protected get indexBatchSize() {\n return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n }\n\n protected get maxIndexSize() {\n return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n }\n\n protected all(order: Order = 'asc', offset?: Hash) {\n return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n }\n\n protected allAsc(offset?: Hash) {\n const pairs = this.payloadPairs\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected allDesc(offset?: Hash) {\n const pairs = this.payloadPairs.reverse()\n const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n return this.payloadPairs.slice(startIndex).map(([payload]) => payload)\n }\n\n protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n if (!this._archivistInstance) {\n const archivist = await super.archivistInstance()\n if (required && !archivist) {\n throw new Error('Failed to find archivist')\n }\n archivist?.on('inserted', this.onArchivistInsert)\n this._archivistInstance = archivist\n }\n return this._archivistInstance\n }\n\n protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n PayloadDivinerQueryPayload<EmptyObject, Hash>\n >\n\n await this.updateIndex()\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n let all: WithMeta<TOut>[] = this.all(order, offset)\n if (all) {\n if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n if (Object.keys(props).length > 0) {\n const additionalFilterCriteria = Object.entries(props)\n for (const [prop, filter] of additionalFilterCriteria) {\n const property = prop as keyof TOut\n all =\n Array.isArray(filter) ?\n all.filter((payload) =>\n filter.every((value) => {\n const prop = payload?.[property]\n //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n return Array.isArray(prop) && prop.includes?.(value)\n }),\n )\n : all.filter((payload) => payload?.[property] === filter)\n }\n }\n return limit ? all.slice(0, limit) : all\n } else {\n throw new Error('Archivist does not support \"all\"')\n }\n }\n\n protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {\n //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))\n forget(this.updateIndex())\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n const archivist = await this.archivistInstance(true)\n archivist.off('inserted', this.onArchivistInsert)\n return await super.stopHandler()\n }\n\n //index any new payloads\n protected async updateIndex() {\n await this._updatePayloadPairsMutex.runExclusive(async () => {\n const archivist = await this.archivistInstance(true)\n let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n while (newPayloads.length > 0) {\n console.log('newPayloads', newPayloads)\n this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n await this.indexPayloads(newPayloads)\n newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n }\n })\n }\n\n private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n this.payloadPairs.push(...pairs)\n return assertEx(pairs.at(-1))[1]\n }\n}\n"],"mappings":";;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,8BAEAC,kCAGK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,yBAAN,MAAMA,+BASHC,eAAAA;EAGEC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBT;EACvC;EAEA,IAAcW,eAAe;AAC3B,WAAO,KAAKD,OAAOC,gBAAgBV;EACrC;EAEUW,IAAIC,QAAe,OAAOC,QAAe;AACjD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKb;AACnB,UAAMc,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ,KAAKb,aAAaoB,QAAO;AACvC,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAO,KAAKV,aAAaiB,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EAChE;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKrB,oBAAoB;AAC5B,YAAMsB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/B,WAAKzB,qBAAqBsB;IAC5B;AACA,WAAO,KAAKtB;EACd;EAEA,MAAyB0B,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,kCAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAAS9B,OAAO+B,OAAO9B,QAAQ,GAAG+B,MAAAA,IAAUX;AAC1E,QAAItB,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAI+B,mCAASN;AAAQzB,cAAMA,IAAIsB,OAAO,CAACX,YAAYoB,QAAQG,SAASvB,QAAQmB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjBvC,gBACEyC,MAAMC,QAAQpB,OAAAA,IACZtB,IAAIsB,OAAO,CAACX,YACVW,QAAOqB,MAAM,CAACC,UAAAA;AAnG9B;AAoGkB,kBAAML,QAAO5B,mCAAU6B;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEF5C,IAAIsB,OAAO,CAACX,aAAYA,mCAAU6B,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQhC,IAAIS,MAAM,GAAGuB,KAAAA,IAAShC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUE,oBAAyE,MAAA;AAEjF2B,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMhC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUiC,IAAI,YAAY,KAAK9B,iBAAiB;AAChD,WAAO,MAAM,MAAM4B,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKhC,yBAAyBsD,aAAa,YAAA;AAC/C,YAAMlC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIqC,cAAe,MAAMnC,UAAUoC,KAAK;QAAEnB,OAAO;QAAK9B,QAAQ,KAAKR;MAAa,CAAA;AAChF,aAAOwD,YAAYzB,SAAS,GAAG;AAC7B2B,gBAAQC,IAAI,eAAeH,WAAAA;AAC3B,aAAKxD,eAAe,MAAM4D,eAAe9C,KAAKgB,SAAS0B,YAAYK,GAAG,EAAC,CAAA,CAAA;AACvE/B,iBAAS,KAAKhC,aAAaiC,SAASyB,YAAYzB,UAAU,KAAK1B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAKyD,cAAcN,WAAAA;AACzBA,sBAAe,MAAMnC,UAAUoC,KAAK;UAAEnB,OAAO;UAAK9B,QAAQ,KAAKR;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAc8D,cAAcpC,UAA2C;AACrE,UAAMf,QAAQ,MAAMiD,eAAeG,UAAUrC,QAAAA;AAC7C,SAAK5B,aAAakE,KAAI,GAAIrD,KAAAA;AAC1B,WAAOmB,SAASnB,MAAMkD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAlHUhE;AACR,cAVWD,wBAUKqE,iBAAgB;EAACC;;AAV5B,IAAMtE,wBAAN;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadDivinerConfigSchema","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDiviner","PayloadDiviner","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInsert","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","runExclusive","newPayloads","next","console","log","PayloadBuilder","at","indexPayloads","hashPairs","push","configSchemas","PayloadDivinerConfigSchema"]}
package/package.json CHANGED
@@ -14,22 +14,22 @@
14
14
  "@xylabs/forget": "^3.1.8",
15
15
  "@xylabs/hex": "^3.1.8",
16
16
  "@xylabs/object": "^3.1.8",
17
- "@xyo-network/archivist-model": "~2.95.7",
18
- "@xyo-network/diviner-model": "~2.95.7",
19
- "@xyo-network/diviner-payload-abstract": "~2.95.7",
20
- "@xyo-network/diviner-payload-model": "~2.95.7",
21
- "@xyo-network/module-events": "~2.95.7",
22
- "@xyo-network/payload-builder": "~2.95.7",
23
- "@xyo-network/payload-model": "~2.95.7",
17
+ "@xyo-network/archivist-model": "~2.95.9",
18
+ "@xyo-network/diviner-model": "~2.95.9",
19
+ "@xyo-network/diviner-payload-abstract": "~2.95.9",
20
+ "@xyo-network/diviner-payload-model": "~2.95.9",
21
+ "@xyo-network/module-events": "~2.95.9",
22
+ "@xyo-network/payload-builder": "~2.95.9",
23
+ "@xyo-network/payload-model": "~2.95.9",
24
24
  "async-mutex": "^0.5.0"
25
25
  },
26
26
  "devDependencies": {
27
27
  "@xylabs/ts-scripts-yarn3": "^3.9.1",
28
28
  "@xylabs/tsconfig": "^3.9.1",
29
- "@xyo-network/account": "~2.95.7",
30
- "@xyo-network/archivist-memory": "~2.95.7",
31
- "@xyo-network/node-memory": "~2.95.7",
32
- "@xyo-network/payload-builder": "~2.95.7",
29
+ "@xyo-network/account": "~2.95.9",
30
+ "@xyo-network/archivist-memory": "~2.95.9",
31
+ "@xyo-network/node-memory": "~2.95.9",
32
+ "@xyo-network/payload-builder": "~2.95.9",
33
33
  "typescript": "^5.4.5"
34
34
  },
35
35
  "description": "Primary SDK for using XYO Protocol 2.0",
@@ -71,6 +71,6 @@
71
71
  "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
72
72
  },
73
73
  "sideEffects": false,
74
- "version": "2.95.7",
74
+ "version": "2.95.9",
75
75
  "type": "module"
76
76
  }
package/src/Diviner.ts CHANGED
@@ -35,6 +35,7 @@ export class GenericPayloadDiviner<
35
35
  protected payloadPairs: [WithMeta<TOut>, Hash][] = []
36
36
 
37
37
  private _archivistInstance?: ArchivistInstance
38
+ private _indexOffset?: Hash
38
39
  private _updatePayloadPairsMutex = new Mutex()
39
40
 
40
41
  protected get indexBatchSize() {
@@ -45,23 +46,19 @@ export class GenericPayloadDiviner<
45
46
  return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE
46
47
  }
47
48
 
48
- protected get payloadPairsOffset() {
49
- return this.payloadPairs.at(-1)?.[1]
50
- }
51
-
52
49
  protected all(order: Order = 'asc', offset?: Hash) {
53
50
  return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)
54
51
  }
55
52
 
56
53
  protected allAsc(offset?: Hash) {
57
54
  const pairs = this.payloadPairs
58
- const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0
55
+ const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1
59
56
  return this.payloadPairs.slice(startIndex).map(([payload]) => payload)
60
57
  }
61
58
 
62
59
  protected allDesc(offset?: Hash) {
63
60
  const pairs = this.payloadPairs.reverse()
64
- const startIndex = offset ? pairs.findIndex(([, hash]) => hash === offset) ?? 0 : 0
61
+ const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1
65
62
  return this.payloadPairs.slice(startIndex).map(([payload]) => payload)
66
63
  }
67
64
 
@@ -115,8 +112,9 @@ export class GenericPayloadDiviner<
115
112
  }
116
113
  }
117
114
 
118
- protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = ({ payloads }) => {
119
- forget(this.indexPayloads(payloads as WithMeta<TOut>[]))
115
+ protected onArchivistInsert: EventListener<ArchivistModuleEventData['inserted']> = () => {
116
+ //forget(this.indexPayloads(payloads as WithMeta<TOut>[]))
117
+ forget(this.updateIndex())
120
118
  }
121
119
 
122
120
  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {
@@ -129,17 +127,20 @@ export class GenericPayloadDiviner<
129
127
  protected async updateIndex() {
130
128
  await this._updatePayloadPairsMutex.runExclusive(async () => {
131
129
  const archivist = await this.archivistInstance(true)
132
- let newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]
130
+ let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]
133
131
  while (newPayloads.length > 0) {
132
+ console.log('newPayloads', newPayloads)
133
+ this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))
134
134
  assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')
135
135
  await this.indexPayloads(newPayloads)
136
- newPayloads = (await archivist.next({ limit: 100, offset: this.payloadPairsOffset })) as WithMeta<TOut>[]
136
+ newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]
137
137
  }
138
138
  })
139
139
  }
140
140
 
141
- private async indexPayloads(payloads: WithMeta<TOut>[]) {
141
+ private async indexPayloads(payloads: WithMeta<TOut>[]): Promise<Hash> {
142
142
  const pairs = await PayloadBuilder.hashPairs(payloads)
143
143
  this.payloadPairs.push(...pairs)
144
+ return assertEx(pairs.at(-1))[1]
144
145
  }
145
146
  }