@xyo-network/diviner-indexing-memory 2.90.2 → 2.90.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. package/dist/browser/Diviner.d.cts +26 -2
  2. package/dist/browser/Diviner.d.cts.map +1 -1
  3. package/dist/browser/Diviner.d.mts +26 -2
  4. package/dist/browser/Diviner.d.mts.map +1 -1
  5. package/dist/browser/Diviner.d.ts +26 -2
  6. package/dist/browser/Diviner.d.ts.map +1 -1
  7. package/dist/browser/index.cjs +3 -3
  8. package/dist/browser/index.cjs.map +1 -1
  9. package/dist/browser/index.js +4 -4
  10. package/dist/browser/index.js.map +1 -1
  11. package/dist/node/Diviner.d.cts +26 -2
  12. package/dist/node/Diviner.d.cts.map +1 -1
  13. package/dist/node/Diviner.d.mts +26 -2
  14. package/dist/node/Diviner.d.mts.map +1 -1
  15. package/dist/node/Diviner.d.ts +26 -2
  16. package/dist/node/Diviner.d.ts.map +1 -1
  17. package/dist/node/index.cjs +3 -3
  18. package/dist/node/index.cjs.map +1 -1
  19. package/dist/node/index.js +4 -4
  20. package/dist/node/index.js.map +1 -1
  21. package/package.json +14 -14
  22. package/src/Diviner.ts +7 -7
package/dist/browser/Diviner.d.cts CHANGED
@@ -4,7 +4,7 @@ import { IndexingDivinerConfig, IndexingDivinerParams, IndexingDivinerStage, Ind
4
4
  import { DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model';
5
5
  import { DivinerWrapper } from '@xyo-network/diviner-wrapper';
6
6
  import { ModuleState } from '@xyo-network/module-model';
7
- import { Payload } from '@xyo-network/payload-model';
7
+ import { Payload, WithMeta } from '@xyo-network/payload-model';
8
8
  export type ConfigStoreKey = 'indexStore' | 'stateStore';
9
9
  export type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>;
10
10
  export declare class IndexingDiviner<TParams extends IndexingDivinerParams = IndexingDivinerParams, TIn extends Payload = Payload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {
@@ -27,7 +27,7 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
27
27
  * @param nextState The state to commit
28
28
  */
29
29
  protected commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void>;
30
- protected divineHandler(payloads?: TIn[]): Promise<TOut[]>;
30
+ protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
31
31
  /**
32
32
  * Retrieves the archivist for the specified store
33
33
  * @param store The store to retrieve the archivist for
@@ -47,6 +47,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
47
47
  readonly security?: {
48
48
  readonly allowAnonymous?: boolean | undefined;
49
49
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
50
+ /**
51
+ * Works via batched iteration of the source archivist to populate the index.
52
+ * @returns A promise that resolves when the background process is complete
53
+ */
50
54
  readonly disallowed?: Record<string, string[]> | undefined;
51
55
  } | undefined;
52
56
  readonly sign?: boolean | undefined;
@@ -64,6 +68,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
64
68
  readonly security?: {
65
69
  readonly allowAnonymous?: boolean | undefined;
66
70
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
71
+ /**
72
+ * Works via batched iteration of the source archivist to populate the index.
73
+ * @returns A promise that resolves when the background process is complete
74
+ */
67
75
  readonly disallowed?: Record<string, string[]> | undefined;
68
76
  } | undefined;
69
77
  readonly sign?: boolean | undefined;
@@ -98,6 +106,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
98
106
  readonly security?: {
99
107
  readonly allowAnonymous?: boolean | undefined;
100
108
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
109
+ /**
110
+ * Works via batched iteration of the source archivist to populate the index.
111
+ * @returns A promise that resolves when the background process is complete
112
+ */
101
113
  readonly disallowed?: Record<string, string[]> | undefined;
102
114
  } | undefined;
103
115
  readonly sign?: boolean | undefined;
@@ -115,6 +127,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
115
127
  readonly security?: {
116
128
  readonly allowAnonymous?: boolean | undefined;
117
129
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
130
+ /**
131
+ * Works via batched iteration of the source archivist to populate the index.
132
+ * @returns A promise that resolves when the background process is complete
133
+ */
118
134
  readonly disallowed?: Record<string, string[]> | undefined;
119
135
  } | undefined;
120
136
  readonly sign?: boolean | undefined;
@@ -202,6 +218,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
202
218
  readonly security?: {
203
219
  readonly allowAnonymous?: boolean | undefined;
204
220
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
221
+ /**
222
+ * Works via batched iteration of the source archivist to populate the index.
223
+ * @returns A promise that resolves when the background process is complete
224
+ */
205
225
  readonly disallowed?: Record<string, string[]> | undefined;
206
226
  } | undefined;
207
227
  readonly sign?: boolean | undefined;
@@ -219,6 +239,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
219
239
  readonly security?: {
220
240
  readonly allowAnonymous?: boolean | undefined;
221
241
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
242
+ /**
243
+ * Works via batched iteration of the source archivist to populate the index.
244
+ * @returns A promise that resolves when the background process is complete
245
+ */
222
246
  readonly disallowed?: Record<string, string[]> | undefined;
223
247
  } | undefined;
224
248
  readonly sign?: boolean | undefined;
package/dist/browser/Diviner.d.cts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAyD,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAEjI,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAE9D,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAsBvF;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBApEvD;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;IAuEH;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBA/EjE;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IAkFH;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBAxG5D;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IA2GH;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
package/dist/browser/Diviner.d.mts CHANGED
@@ -4,7 +4,7 @@ import { IndexingDivinerConfig, IndexingDivinerParams, IndexingDivinerStage, Ind
4
4
  import { DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model';
5
5
  import { DivinerWrapper } from '@xyo-network/diviner-wrapper';
6
6
  import { ModuleState } from '@xyo-network/module-model';
7
- import { Payload } from '@xyo-network/payload-model';
7
+ import { Payload, WithMeta } from '@xyo-network/payload-model';
8
8
  export type ConfigStoreKey = 'indexStore' | 'stateStore';
9
9
  export type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>;
10
10
  export declare class IndexingDiviner<TParams extends IndexingDivinerParams = IndexingDivinerParams, TIn extends Payload = Payload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {
@@ -27,7 +27,7 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
27
27
  * @param nextState The state to commit
28
28
  */
29
29
  protected commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void>;
30
- protected divineHandler(payloads?: TIn[]): Promise<TOut[]>;
30
+ protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
31
31
  /**
32
32
  * Retrieves the archivist for the specified store
33
33
  * @param store The store to retrieve the archivist for
@@ -47,6 +47,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
47
47
  readonly security?: {
48
48
  readonly allowAnonymous?: boolean | undefined;
49
49
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
50
+ /**
51
+ * Works via batched iteration of the source archivist to populate the index.
52
+ * @returns A promise that resolves when the background process is complete
53
+ */
50
54
  readonly disallowed?: Record<string, string[]> | undefined;
51
55
  } | undefined;
52
56
  readonly sign?: boolean | undefined;
@@ -64,6 +68,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
64
68
  readonly security?: {
65
69
  readonly allowAnonymous?: boolean | undefined;
66
70
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
71
+ /**
72
+ * Works via batched iteration of the source archivist to populate the index.
73
+ * @returns A promise that resolves when the background process is complete
74
+ */
67
75
  readonly disallowed?: Record<string, string[]> | undefined;
68
76
  } | undefined;
69
77
  readonly sign?: boolean | undefined;
@@ -98,6 +106,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
98
106
  readonly security?: {
99
107
  readonly allowAnonymous?: boolean | undefined;
100
108
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
109
+ /**
110
+ * Works via batched iteration of the source archivist to populate the index.
111
+ * @returns A promise that resolves when the background process is complete
112
+ */
101
113
  readonly disallowed?: Record<string, string[]> | undefined;
102
114
  } | undefined;
103
115
  readonly sign?: boolean | undefined;
@@ -115,6 +127,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
115
127
  readonly security?: {
116
128
  readonly allowAnonymous?: boolean | undefined;
117
129
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
130
+ /**
131
+ * Works via batched iteration of the source archivist to populate the index.
132
+ * @returns A promise that resolves when the background process is complete
133
+ */
118
134
  readonly disallowed?: Record<string, string[]> | undefined;
119
135
  } | undefined;
120
136
  readonly sign?: boolean | undefined;
@@ -202,6 +218,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
202
218
  readonly security?: {
203
219
  readonly allowAnonymous?: boolean | undefined;
204
220
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
221
+ /**
222
+ * Works via batched iteration of the source archivist to populate the index.
223
+ * @returns A promise that resolves when the background process is complete
224
+ */
205
225
  readonly disallowed?: Record<string, string[]> | undefined;
206
226
  } | undefined;
207
227
  readonly sign?: boolean | undefined;
@@ -219,6 +239,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
219
239
  readonly security?: {
220
240
  readonly allowAnonymous?: boolean | undefined;
221
241
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
242
+ /**
243
+ * Works via batched iteration of the source archivist to populate the index.
244
+ * @returns A promise that resolves when the background process is complete
245
+ */
222
246
  readonly disallowed?: Record<string, string[]> | undefined;
223
247
  } | undefined;
224
248
  readonly sign?: boolean | undefined;
package/dist/browser/Diviner.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAyD,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAEjI,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAE9D,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAsBvF;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBApEvD;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;IAuEH;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBA/EjE;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IAkFH;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBAxG5D;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IA2GH;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
package/dist/browser/Diviner.d.ts CHANGED
@@ -4,7 +4,7 @@ import { IndexingDivinerConfig, IndexingDivinerParams, IndexingDivinerStage, Ind
4
4
  import { DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model';
5
5
  import { DivinerWrapper } from '@xyo-network/diviner-wrapper';
6
6
  import { ModuleState } from '@xyo-network/module-model';
7
- import { Payload } from '@xyo-network/payload-model';
7
+ import { Payload, WithMeta } from '@xyo-network/payload-model';
8
8
  export type ConfigStoreKey = 'indexStore' | 'stateStore';
9
9
  export type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>;
10
10
  export declare class IndexingDiviner<TParams extends IndexingDivinerParams = IndexingDivinerParams, TIn extends Payload = Payload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {
@@ -27,7 +27,7 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
27
27
  * @param nextState The state to commit
28
28
  */
29
29
  protected commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void>;
30
- protected divineHandler(payloads?: TIn[]): Promise<TOut[]>;
30
+ protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
31
31
  /**
32
32
  * Retrieves the archivist for the specified store
33
33
  * @param store The store to retrieve the archivist for
@@ -47,6 +47,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
47
47
  readonly security?: {
48
48
  readonly allowAnonymous?: boolean | undefined;
49
49
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
50
+ /**
51
+ * Works via batched iteration of the source archivist to populate the index.
52
+ * @returns A promise that resolves when the background process is complete
53
+ */
50
54
  readonly disallowed?: Record<string, string[]> | undefined;
51
55
  } | undefined;
52
56
  readonly sign?: boolean | undefined;
@@ -64,6 +68,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
64
68
  readonly security?: {
65
69
  readonly allowAnonymous?: boolean | undefined;
66
70
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
71
+ /**
72
+ * Works via batched iteration of the source archivist to populate the index.
73
+ * @returns A promise that resolves when the background process is complete
74
+ */
67
75
  readonly disallowed?: Record<string, string[]> | undefined;
68
76
  } | undefined;
69
77
  readonly sign?: boolean | undefined;
@@ -98,6 +106,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
98
106
  readonly security?: {
99
107
  readonly allowAnonymous?: boolean | undefined;
100
108
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
109
+ /**
110
+ * Works via batched iteration of the source archivist to populate the index.
111
+ * @returns A promise that resolves when the background process is complete
112
+ */
101
113
  readonly disallowed?: Record<string, string[]> | undefined;
102
114
  } | undefined;
103
115
  readonly sign?: boolean | undefined;
@@ -115,6 +127,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
115
127
  readonly security?: {
116
128
  readonly allowAnonymous?: boolean | undefined;
117
129
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
130
+ /**
131
+ * Works via batched iteration of the source archivist to populate the index.
132
+ * @returns A promise that resolves when the background process is complete
133
+ */
118
134
  readonly disallowed?: Record<string, string[]> | undefined;
119
135
  } | undefined;
120
136
  readonly sign?: boolean | undefined;
@@ -202,6 +218,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
202
218
  readonly security?: {
203
219
  readonly allowAnonymous?: boolean | undefined;
204
220
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
221
+ /**
222
+ * Works via batched iteration of the source archivist to populate the index.
223
+ * @returns A promise that resolves when the background process is complete
224
+ */
205
225
  readonly disallowed?: Record<string, string[]> | undefined;
206
226
  } | undefined;
207
227
  readonly sign?: boolean | undefined;
@@ -219,6 +239,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
219
239
  readonly security?: {
220
240
  readonly allowAnonymous?: boolean | undefined;
221
241
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
242
+ /**
243
+ * Works via batched iteration of the source archivist to populate the index.
244
+ * @returns A promise that resolves when the background process is complete
245
+ */
222
246
  readonly disallowed?: Record<string, string[]> | undefined;
223
247
  } | undefined;
224
248
  readonly sign?: boolean | undefined;
package/dist/browser/Diviner.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAyD,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAEjI,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAE9D,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAsBvF;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBApEvD;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;IAuEH;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBA/EjE;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IAkFH;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBAxG5D;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IA2GH;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
package/dist/browser/index.cjs CHANGED
@@ -77,8 +77,8 @@ var IndexingDiviner = class extends import_diviner_abstract.AbstractDiviner {
77
77
  const results = lastState ? await indexCandidateDiviner.divine([
78
78
  lastState
79
79
  ]) : await indexCandidateDiviner.divine();
80
- const nextState = results.find(import_module_model.isModuleState);
81
- const indexCandidates = results.filter((x) => !(0, import_module_model.isModuleState)(x));
80
+ const nextState = results.find(import_module_model.isModuleStateWithMeta);
81
+ const indexCandidates = results.filter((x) => !(0, import_module_model.isModuleStateWithMeta)(x));
82
82
  const toIndexTransformDiviner = await this.getIndexingDivinerStage("indexCandidateToIndexDiviner");
83
83
  const indexes = await toIndexTransformDiviner.divine(indexCandidates);
84
84
  const indexArchivist = await this.getArchivistForStore("indexStore");
@@ -120,7 +120,7 @@ var IndexingDiviner = class extends import_diviner_abstract.AbstractDiviner {
120
120
  ])));
121
121
  return response.flat();
122
122
  }))).flat();
123
- return results;
123
+ return await Promise.all(results.map((result) => import_payload_builder.PayloadBuilder.build(result)));
124
124
  }
125
125
  /**
126
126
  * Retrieves the archivist for the specified store
package/dist/browser/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await (await new BoundWitnessBuilder().payload(nextState)).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,mBAA6C;AAC7C,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,8BAAgC;AAChC,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA+E;AAC/E,6BAA+B;;;;;;;;;;;;AAO/B,IAAMA,aAAa;AAGZ,IAAMC,kBAAN,cAKGC,wCAAAA;SAAAA;;;EACR,OAAyBC,qBAAqB;EAC9C,OAAyBC,gBAA0B;IAACC;IAA6BC;;EAEzEC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,iCAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,KAACH,mCAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;AAExE,QAAIA,UAAUY,MAAMC,WAAW,KAAKzB,YAAYwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,OAAO,MAAM,IAAIC,gDAAAA,EAAsBC,QAAQjB,SAAAA,GAAYkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AACnG,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;AACvD,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQrB,WAAW,MAAM,GAAGjC,UAAAA,gBAA0BsD,KAAAA,0BAA+B;AACzH,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,eAAoBC,IAAAA,GAAO;AACpH,WAAOI,0CAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;AACjE,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQQ,qBAAqB,MAAM,GAAG9D,UAAAA,gBAA0BsD,KAAAA,oCAAyC;AAC7I,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,yBAA8BC,IAAAA,GAAO;AAC9H,WAAOQ,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;AACvE,UAAMC,oBAAgBT,wBACpB,KAAK9C,QAAQwD,wBAAwBF,SAAAA,GACrC,MAAM,GAAGhE,UAAAA,sCAAgDgE,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,eAAOT,4BAASW,wCAAkBV,GAAAA,GAAM,MAAM,GAAGzD,UAAAA,kDAA4DgE,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;AAC5D,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQc,gBAAgB,MAAM,GAAGpE,UAAAA,gBAA0BsD,KAAAA,+BAAoC;AACnI,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,oBAAyBC,IAAAA,GAAO;AACzH,WAAOQ,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,sCAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,cAAIG,0CAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAUN,aAAaJ,kBAAkBU,MAAMH,KAAAA,MAAWN,wCAAoBG,aAAaO,eAAeD,MAAMH,KAAAA,IAASE,MAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,iCAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,cAAU0F,2BAAa,YAAA;AAC1B,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAAS8F,2CAAe,KAAK9F,OAAO;AAC7C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA/MaV,kBAAAA,aAAAA;MADZsG,qCAAAA;GACYtG,eAAAA;","names":["moduleName","IndexingDiviner","AbstractDiviner","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","creatableModule"]}
1
+ {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, isModuleStateWithMeta, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleStateWithMeta<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleStateWithMeta(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await (await new BoundWitnessBuilder().payload(nextState)).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<WithMeta<TOut>[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return (await Promise.all(results.map((result) => PayloadBuilder.build(result)))) as WithMeta<TOut>[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload as WithMeta<ModuleState<IndexingDivinerState>>\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,mBAA6C;AAC7C,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,8BAAgC;AAChC,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAAsG;AACtG,6BAA+B;;;;;;;;;;;;AAO/B,IAAMA,aAAa;AAGZ,IAAMC,kBAAN,cAKGC,wCAAAA;SAAAA;;;EACR,OAAyBC,qBAAqB;EAC9C,OAAyBC,gBAA0B;IAACC;IAA6BC;;EAEzEC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,yCAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,KAACH,2CAAsBG,CAAAA,CAAAA;AAErE,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;AAExE,QAAIA,UAAUY,MAAMC,WAAW,KAAKzB,YAAYwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,OAAO,MAAM,IAAIC,gDAAAA,EAAsBC,QAAQjB,SAAAA,GAAYkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AACnG,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAA+B;AACtF,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAQ,MAAMN,QAAQC,IAAI9B,QAAQ+B,IAAI,CAACM,WAAWC,sCAAehB,MAAMe,MAAAA,CAAAA,CAAAA;EACzE;;;;;;EAOA,MAAgB1B,qBAAqB4B,OAAoB;AACvD,UAAMC,WAAOC,wBAAS,KAAKhD,SAAS8C,KAAAA,GAAQvB,WAAW,MAAM,GAAGjC,UAAAA,gBAA0BwD,KAAAA,0BAA+B;AACzH,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGzD,UAAAA,uBAAiCwD,KAAAA,eAAoBC,IAAAA,GAAO;AACpH,WAAOI,0CAAiBC,KAAKH,KAAK,KAAKrB,OAAO;EAChD;;;;;;EAOA,MAAgByB,+BAA+BP,OAAoB;AACjE,UAAMC,WAAOC,wBAAS,KAAKhD,SAAS8C,KAAAA,GAAQQ,qBAAqB,MAAM,GAAGhE,UAAAA,gBAA0BwD,KAAAA,oCAAyC;AAC7I,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGzD,UAAAA,uBAAiCwD,KAAAA,yBAA8BC,IAAAA,GAAO;AAC9H,WAAOQ,sCAAeH,KAAKH,KAAK,KAAKrB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBkD,WAAiC;AACvE,UAAMC,oBAAgBT,wBACpB,KAAKhD,QAAQ0D,wBAAwBF,SAAAA,GACrC,MAAM,GAAGlE,UAAAA,sCAAgDkE,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,eAAOT,4BAASW,wCAAkBV,GAAAA,GAAM,MAAM,GAAG3D,UAAAA,kDAA4DkE,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBvB,0BAA0Ba,OAAoB;AAC5D,UAAMC,WAAOC,wBAAS,KAAKhD,SAAS8C,KAAAA,GAAQc,gBAAgB,MAAM,GAAGtE,UAAAA,gBAA0BwD,KAAAA,+BAAoC;AACnI,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGzD,UAAAA,uBAAiCwD,KAAAA,oBAAyBC,IAAAA,GAAO;AACzH,WAAOQ,sCAAeH,KAAKH,KAAK,KAAKrB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAIgE,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIlB,sCAAgD;MAAEmB,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKvC,QAAQuC;MACtBC,OAAO;MACP9C,QAAQ;MACR+C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACC1C,MAAK;AACR,UAAM2C,iBAAiB,MAAMV,QAAQtD,OAAO;MAACuD;KAAM;AACnD,QAAIS,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,cAAIG,0CAAeD,YAAAA,GAAe;AAEhCb,eAAOa,aAAaE,UACjBtC,IAAI,CAAC6B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1ChE,OAAO,CAAC,EAAEsD,QAAO,MAAOA,YAAY,KAAKvC,QAAQuC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAUN,aAAaJ,kBAAkBU,MAAMH,KAAAA,MAAWN,wCAAoBG,aAAaO,eAAeD,MAAMH,KAAAA,IAASE,MAChI,EAAA;MAEN;IACF;AAGA,QAAIlB,MAAM;AAER,YAAMtC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU2D,IAAI;QAACrB;OAAK,GAAGnD,KAAKyE,iCAAAA;AACnD,UAAIzD,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAO0D;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAK1F,SAAS;AAChB2F,mBAAa,KAAK3F,OAAO;AACzB,WAAKA,UAAUsF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKxF,cAAU4F,2BAAa,YAAA;AAC1B,UAAI;AACF,cAAMtD,QAAQc,QAAO;AACrB,cAAM,KAAKhD,iBAAgB;MAC7B,SAASyF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK7F;AAASgG,2CAAe,KAAKhG,OAAO;AAC7C,aAAKA,UAAUsF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKrF,aAAa;EACvB;AACF;AA/MaV,kBAAAA,aAAAA;MADZwG,qCAAAA;GACYxG,eAAAA;","names":["moduleName","IndexingDiviner","AbstractDiviner","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleStateWithMeta","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","result","PayloadBuilder","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","isModuleState","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","creatableModule"]}
package/dist/browser/index.js CHANGED
@@ -12,7 +12,7 @@ import { BoundWitnessDivinerQuerySchema } from "@xyo-network/diviner-boundwitnes
12
12
  import { IndexingDivinerConfigSchema } from "@xyo-network/diviner-indexing-model";
13
13
  import { asDivinerInstance, DivinerConfigSchema } from "@xyo-network/diviner-model";
14
14
  import { DivinerWrapper } from "@xyo-network/diviner-wrapper";
15
- import { creatableModule, isModuleState, ModuleStateSchema } from "@xyo-network/module-model";
15
+ import { creatableModule, isModuleState, isModuleStateWithMeta, ModuleStateSchema } from "@xyo-network/module-model";
16
16
  import { PayloadBuilder } from "@xyo-network/payload-builder";
17
17
  function _ts_decorate(decorators, target, key, desc) {
18
18
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
@@ -53,8 +53,8 @@ var IndexingDiviner = class extends AbstractDiviner {
53
53
  const results = lastState ? await indexCandidateDiviner.divine([
54
54
  lastState
55
55
  ]) : await indexCandidateDiviner.divine();
56
- const nextState = results.find(isModuleState);
57
- const indexCandidates = results.filter((x) => !isModuleState(x));
56
+ const nextState = results.find(isModuleStateWithMeta);
57
+ const indexCandidates = results.filter((x) => !isModuleStateWithMeta(x));
58
58
  const toIndexTransformDiviner = await this.getIndexingDivinerStage("indexCandidateToIndexDiviner");
59
59
  const indexes = await toIndexTransformDiviner.divine(indexCandidates);
60
60
  const indexArchivist = await this.getArchivistForStore("indexStore");
@@ -96,7 +96,7 @@ var IndexingDiviner = class extends AbstractDiviner {
96
96
  ])));
97
97
  return response.flat();
98
98
  }))).flat();
99
- return results;
99
+ return await Promise.all(results.map((result) => PayloadBuilder.build(result)));
100
100
  }
101
101
  /**
102
102
  * Retrieves the archivist for the specified store
package/dist/browser/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await (await new BoundWitnessBuilder().payload(nextState)).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,gBAAgBC,oBAAoB;AAC7C,SAASC,wBAAwB;AACjC,SAASC,2BAA2B;AACpC,SAASC,sBAAsB;AAC/B,SAASC,uBAAuB;AAChC,SAA0CC,sCAAsC;AAChF,SAEEC,mCAIK;AACP,SAASC,mBAAmBC,2BAAkE;AAC9F,SAASC,sBAAsB;AAC/B,SAASC,iBAAiBC,eAA4BC,yBAAyB;AAC/E,SAASC,sBAAsB;;;;;;;;;;;;AAO/B,IAAMC,aAAa;AAGZ,IAAMC,kBAAN,cAKGC,gBAAAA;SAAAA;;;EACR,OAAyBC,qBAAqB;EAC9C,OAAyBC,gBAA0B;IAACC;IAA6BC;;EAEzEC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,aAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,CAACH,cAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;AAExE,QAAIA,UAAUY,MAAMC,WAAW,KAAKzB,YAAYwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,OAAO,MAAM,IAAIC,oBAAAA,EAAsBC,QAAQjB,SAAAA,GAAYkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AACnG,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;AACvD,UAAMC,OAAOC,SAAS,KAAK9C,SAAS4C,KAAAA,GAAQrB,WAAW,MAAM,GAAGjC,UAAAA,gBAA0BsD,KAAAA,0BAA+B;AACzH,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,eAAoBC,IAAAA,GAAO;AACpH,WAAOI,iBAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;AACjE,UAAMC,OAAOC,SAAS,KAAK9C,SAAS4C,KAAAA,GAAQQ,qBAAqB,MAAM,GAAG9D,UAAAA,gBAA0BsD,KAAAA,oCAAyC;AAC7I,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,yBAA8BC,IAAAA,GAAO;AAC9H,WAAOQ,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;AACvE,UAAMC,gBAAgBT,SACpB,KAAK9C,QAAQwD,wBAAwBF,SAAAA,GACrC,MAAM,GAAGhE,UAAAA,sCAAgDgE,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,WAAOT,SAASW,kBAAkBV,GAAAA,GAAM,MAAM,GAAGzD,UAAAA,kDAA4DgE,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;AAC5D,UAAMC,OAAOC,SAAS,KAAK9C,SAAS4C,KAAAA,GAAQc,gBAAgB,MAAM,GAAGpE,UAAAA,gBAA0BsD,KAAAA,+BAAoC;AACnI,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,oBAAyBC,IAAAA,GAAO;AACzH,WAAOQ,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,eAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,UAAIG,eAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAUN,aAAaJ,kBAAkBU,MAAMH,KAAAA,MAAWN,oBAAoBG,aAAaO,eAAeD,MAAMH,KAAAA,IAASE,MAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,aAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,UAAU0F,aAAa,YAAA;AAC1B,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAAS8F,yBAAe,KAAK9F,OAAO;AAC7C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA/MaV,kBAAAA,aAAAA;EADZsG,gBAAAA;GACYtG,eAAAA;","names":["assertEx","clearTimeoutEx","setTimeoutEx","ArchivistWrapper","BoundWitnessBuilder","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","IndexingDivinerConfigSchema","asDivinerInstance","DivinerConfigSchema","DivinerWrapper","creatableModule","isModuleState","ModuleStateSchema","PayloadBuilder","moduleName","IndexingDiviner","AbstractDiviner","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","creatableModule"]}
1
+ {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, isModuleStateWithMeta, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleStateWithMeta<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleStateWithMeta(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await (await new BoundWitnessBuilder().payload(nextState)).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<WithMeta<TOut>[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return (await Promise.all(results.map((result) => PayloadBuilder.build(result)))) as WithMeta<TOut>[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload as WithMeta<ModuleState<IndexingDivinerState>>\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,gBAAgBC,oBAAoB;AAC7C,SAASC,wBAAwB;AACjC,SAASC,2BAA2B;AACpC,SAASC,sBAAsB;AAC/B,SAASC,uBAAuB;AAChC,SAA0CC,sCAAsC;AAChF,SAEEC,mCAIK;AACP,SAASC,mBAAmBC,2BAAkE;AAC9F,SAASC,sBAAsB;AAC/B,SAASC,iBAAiBC,eAAeC,uBAAoCC,yBAAyB;AACtG,SAASC,sBAAsB;;;;;;;;;;;;AAO/B,IAAMC,aAAa;AAGZ,IAAMC,kBAAN,cAKGC,gBAAAA;SAAAA;;;EACR,OAAyBC,qBAAqB;EAC9C,OAAyBC,gBAA0B;IAACC;IAA6BC;;EAEzEC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,qBAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,CAACH,sBAAsBG,CAAAA,CAAAA;AAErE,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;AAExE,QAAIA,UAAUY,MAAMC,WAAW,KAAKzB,YAAYwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,OAAO,MAAM,IAAIC,oBAAAA,EAAsBC,QAAQjB,SAAAA,GAAYkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AACnG,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAA+B;AACtF,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAQ,MAAMN,QAAQC,IAAI9B,QAAQ+B,IAAI,CAACM,WAAWC,eAAehB,MAAMe,MAAAA,CAAAA,CAAAA;EACzE;;;;;;EAOA,MAAgB1B,qBAAqB4B,OAAoB;AACvD,UAAMC,OAAOC,SAAS,KAAKhD,SAAS8C,KAAAA,GAAQvB,WAAW,MAAM,GAAGjC,UAAAA,gBAA0BwD,KAAAA,0BAA+B;AACzH,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGzD,UAAAA,uBAAiCwD,KAAAA,eAAoBC,IAAAA,GAAO;AACpH,WAAOI,iBAAiBC,KAAKH,KAAK,KAAKrB,OAAO;EAChD;;;;;;EAOA,MAAgByB,+BAA+BP,OAAoB;AACjE,UAAMC,OAAOC,SAAS,KAAKhD,SAAS8C,KAAAA,GAAQQ,qBAAqB,MAAM,GAAGhE,UAAAA,gBAA0BwD,KAAAA,oCAAyC;AAC7I,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGzD,UAAAA,uBAAiCwD,KAAAA,yBAA8BC,IAAAA,GAAO;AAC9H,WAAOQ,eAAeH,KAAKH,KAAK,KAAKrB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBkD,WAAiC;AACvE,UAAMC,gBAAgBT,SACpB,KAAKhD,QAAQ0D,wBAAwBF,SAAAA,GACrC,MAAM,GAAGlE,UAAAA,sCAAgDkE,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,WAAOT,SAASW,kBAAkBV,GAAAA,GAAM,MAAM,GAAG3D,UAAAA,kDAA4DkE,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBvB,0BAA0Ba,OAAoB;AAC5D,UAAMC,OAAOC,SAAS,KAAKhD,SAAS8C,KAAAA,GAAQc,gBAAgB,MAAM,GAAGtE,UAAAA,gBAA0BwD,KAAAA,+BAAoC;AACnI,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGzD,UAAAA,uBAAiCwD,KAAAA,oBAAyBC,IAAAA,GAAO;AACzH,WAAOQ,eAAeH,KAAKH,KAAK,KAAKrB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAIgE,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIlB,eAAgD;MAAEmB,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKvC,QAAQuC;MACtBC,OAAO;MACP9C,QAAQ;MACR+C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACC1C,MAAK;AACR,UAAM2C,iBAAiB,MAAMV,QAAQtD,OAAO;MAACuD;KAAM;AACnD,QAAIS,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,UAAIG,eAAeD,YAAAA,GAAe;AAEhCb,eAAOa,aAAaE,UACjBtC,IAAI,CAAC6B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1ChE,OAAO,CAAC,EAAEsD,QAAO,MAAOA,YAAY,KAAKvC,QAAQuC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAUN,aAAaJ,kBAAkBU,MAAMH,KAAAA,MAAWN,oBAAoBG,aAAaO,eAAeD,MAAMH,KAAAA,IAASE,MAChI,EAAA;MAEN;IACF;AAGA,QAAIlB,MAAM;AAER,YAAMtC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU2D,IAAI;QAACrB;OAAK,GAAGnD,KAAKyE,aAAAA;AACnD,UAAIzD,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAO0D;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAK1F,SAAS;AAChB2F,mBAAa,KAAK3F,OAAO;AACzB,WAAKA,UAAUsF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKxF,UAAU4F,aAAa,YAAA;AAC1B,UAAI;AACF,cAAMtD,QAAQc,QAAO;AACrB,cAAM,KAAKhD,iBAAgB;MAC7B,SAASyF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK7F;AAASgG,yBAAe,KAAKhG,OAAO;AAC7C,aAAKA,UAAUsF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKrF,aAAa;EACvB;AACF;AA/MaV,kBAAAA,aAAAA;EADZwG,gBAAAA;GACYxG,eAAAA;","names":["assertEx","clearTimeoutEx","setTimeoutEx","ArchivistWrapper","BoundWitnessBuilder","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","IndexingDivinerConfigSchema","asDivinerInstance","DivinerConfigSchema","DivinerWrapper","creatableModule","isModuleState","isModuleStateWithMeta","ModuleStateSchema","PayloadBuilder","moduleName","IndexingDiviner","AbstractDiviner","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleStateWithMeta","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","result","PayloadBuilder","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","isModuleState","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","creatableModule"]}
package/dist/node/Diviner.d.cts CHANGED
@@ -4,7 +4,7 @@ import { IndexingDivinerConfig, IndexingDivinerParams, IndexingDivinerStage, Ind
4
4
  import { DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model';
5
5
  import { DivinerWrapper } from '@xyo-network/diviner-wrapper';
6
6
  import { ModuleState } from '@xyo-network/module-model';
7
- import { Payload } from '@xyo-network/payload-model';
7
+ import { Payload, WithMeta } from '@xyo-network/payload-model';
8
8
  export type ConfigStoreKey = 'indexStore' | 'stateStore';
9
9
  export type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>;
10
10
  export declare class IndexingDiviner<TParams extends IndexingDivinerParams = IndexingDivinerParams, TIn extends Payload = Payload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {
@@ -27,7 +27,7 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
27
27
  * @param nextState The state to commit
28
28
  */
29
29
  protected commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void>;
30
- protected divineHandler(payloads?: TIn[]): Promise<TOut[]>;
30
+ protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
31
31
  /**
32
32
  * Retrieves the archivist for the specified store
33
33
  * @param store The store to retrieve the archivist for
@@ -47,6 +47,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
47
47
  readonly security?: {
48
48
  readonly allowAnonymous?: boolean | undefined;
49
49
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
50
+ /**
51
+ * Works via batched iteration of the source archivist to populate the index.
52
+ * @returns A promise that resolves when the background process is complete
53
+ */
50
54
  readonly disallowed?: Record<string, string[]> | undefined;
51
55
  } | undefined;
52
56
  readonly sign?: boolean | undefined;
@@ -64,6 +68,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
64
68
  readonly security?: {
65
69
  readonly allowAnonymous?: boolean | undefined;
66
70
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
71
+ /**
72
+ * Works via batched iteration of the source archivist to populate the index.
73
+ * @returns A promise that resolves when the background process is complete
74
+ */
67
75
  readonly disallowed?: Record<string, string[]> | undefined;
68
76
  } | undefined;
69
77
  readonly sign?: boolean | undefined;
@@ -98,6 +106,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
98
106
  readonly security?: {
99
107
  readonly allowAnonymous?: boolean | undefined;
100
108
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
109
+ /**
110
+ * Works via batched iteration of the source archivist to populate the index.
111
+ * @returns A promise that resolves when the background process is complete
112
+ */
101
113
  readonly disallowed?: Record<string, string[]> | undefined;
102
114
  } | undefined;
103
115
  readonly sign?: boolean | undefined;
@@ -115,6 +127,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
115
127
  readonly security?: {
116
128
  readonly allowAnonymous?: boolean | undefined;
117
129
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
130
+ /**
131
+ * Works via batched iteration of the source archivist to populate the index.
132
+ * @returns A promise that resolves when the background process is complete
133
+ */
118
134
  readonly disallowed?: Record<string, string[]> | undefined;
119
135
  } | undefined;
120
136
  readonly sign?: boolean | undefined;
@@ -202,6 +218,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
202
218
  readonly security?: {
203
219
  readonly allowAnonymous?: boolean | undefined;
204
220
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
221
+ /**
222
+ * Works via batched iteration of the source archivist to populate the index.
223
+ * @returns A promise that resolves when the background process is complete
224
+ */
205
225
  readonly disallowed?: Record<string, string[]> | undefined;
206
226
  } | undefined;
207
227
  readonly sign?: boolean | undefined;
@@ -219,6 +239,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
219
239
  readonly security?: {
220
240
  readonly allowAnonymous?: boolean | undefined;
221
241
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
242
+ /**
243
+ * Works via batched iteration of the source archivist to populate the index.
244
+ * @returns A promise that resolves when the background process is complete
245
+ */
222
246
  readonly disallowed?: Record<string, string[]> | undefined;
223
247
  } | undefined;
224
248
  readonly sign?: boolean | undefined;
package/dist/node/Diviner.d.cts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAyD,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAEjI,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAE9D,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAsBvF;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBApEvD;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;IAuEH;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBA/EjE;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IAkFH;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBAxG5D;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IA2GH;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
package/dist/node/Diviner.d.mts CHANGED
@@ -4,7 +4,7 @@ import { IndexingDivinerConfig, IndexingDivinerParams, IndexingDivinerStage, Ind
4
4
  import { DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model';
5
5
  import { DivinerWrapper } from '@xyo-network/diviner-wrapper';
6
6
  import { ModuleState } from '@xyo-network/module-model';
7
- import { Payload } from '@xyo-network/payload-model';
7
+ import { Payload, WithMeta } from '@xyo-network/payload-model';
8
8
  export type ConfigStoreKey = 'indexStore' | 'stateStore';
9
9
  export type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>;
10
10
  export declare class IndexingDiviner<TParams extends IndexingDivinerParams = IndexingDivinerParams, TIn extends Payload = Payload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {
@@ -27,7 +27,7 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
27
27
  * @param nextState The state to commit
28
28
  */
29
29
  protected commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void>;
30
- protected divineHandler(payloads?: TIn[]): Promise<TOut[]>;
30
+ protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
31
31
  /**
32
32
  * Retrieves the archivist for the specified store
33
33
  * @param store The store to retrieve the archivist for
@@ -47,6 +47,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
47
47
  readonly security?: {
48
48
  readonly allowAnonymous?: boolean | undefined;
49
49
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
50
+ /**
51
+ * Works via batched iteration of the source archivist to populate the index.
52
+ * @returns A promise that resolves when the background process is complete
53
+ */
50
54
  readonly disallowed?: Record<string, string[]> | undefined;
51
55
  } | undefined;
52
56
  readonly sign?: boolean | undefined;
@@ -64,6 +68,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
64
68
  readonly security?: {
65
69
  readonly allowAnonymous?: boolean | undefined;
66
70
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
71
+ /**
72
+ * Works via batched iteration of the source archivist to populate the index.
73
+ * @returns A promise that resolves when the background process is complete
74
+ */
67
75
  readonly disallowed?: Record<string, string[]> | undefined;
68
76
  } | undefined;
69
77
  readonly sign?: boolean | undefined;
@@ -98,6 +106,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
98
106
  readonly security?: {
99
107
  readonly allowAnonymous?: boolean | undefined;
100
108
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
109
+ /**
110
+ * Works via batched iteration of the source archivist to populate the index.
111
+ * @returns A promise that resolves when the background process is complete
112
+ */
101
113
  readonly disallowed?: Record<string, string[]> | undefined;
102
114
  } | undefined;
103
115
  readonly sign?: boolean | undefined;
@@ -115,6 +127,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
115
127
  readonly security?: {
116
128
  readonly allowAnonymous?: boolean | undefined;
117
129
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
130
+ /**
131
+ * Works via batched iteration of the source archivist to populate the index.
132
+ * @returns A promise that resolves when the background process is complete
133
+ */
118
134
  readonly disallowed?: Record<string, string[]> | undefined;
119
135
  } | undefined;
120
136
  readonly sign?: boolean | undefined;
@@ -202,6 +218,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
202
218
  readonly security?: {
203
219
  readonly allowAnonymous?: boolean | undefined;
204
220
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
221
+ /**
222
+ * Works via batched iteration of the source archivist to populate the index.
223
+ * @returns A promise that resolves when the background process is complete
224
+ */
205
225
  readonly disallowed?: Record<string, string[]> | undefined;
206
226
  } | undefined;
207
227
  readonly sign?: boolean | undefined;
@@ -219,6 +239,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
219
239
  readonly security?: {
220
240
  readonly allowAnonymous?: boolean | undefined;
221
241
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
242
+ /**
243
+ * Works via batched iteration of the source archivist to populate the index.
244
+ * @returns A promise that resolves when the background process is complete
245
+ */
222
246
  readonly disallowed?: Record<string, string[]> | undefined;
223
247
  } | undefined;
224
248
  readonly sign?: boolean | undefined;
package/dist/node/Diviner.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAyD,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAEjI,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAE9D,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAsBvF;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBApEvD;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;IAuEH;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBA/EjE;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IAkFH;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBAxG5D;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IA2GH;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
package/dist/node/Diviner.d.ts CHANGED
@@ -4,7 +4,7 @@ import { IndexingDivinerConfig, IndexingDivinerParams, IndexingDivinerStage, Ind
4
4
  import { DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model';
5
5
  import { DivinerWrapper } from '@xyo-network/diviner-wrapper';
6
6
  import { ModuleState } from '@xyo-network/module-model';
7
- import { Payload } from '@xyo-network/payload-model';
7
+ import { Payload, WithMeta } from '@xyo-network/payload-model';
8
8
  export type ConfigStoreKey = 'indexStore' | 'stateStore';
9
9
  export type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>;
10
10
  export declare class IndexingDiviner<TParams extends IndexingDivinerParams = IndexingDivinerParams, TIn extends Payload = Payload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {
@@ -27,7 +27,7 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
27
27
  * @param nextState The state to commit
28
28
  */
29
29
  protected commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void>;
30
- protected divineHandler(payloads?: TIn[]): Promise<TOut[]>;
30
+ protected divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]>;
31
31
  /**
32
32
  * Retrieves the archivist for the specified store
33
33
  * @param store The store to retrieve the archivist for
@@ -47,6 +47,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
47
47
  readonly security?: {
48
48
  readonly allowAnonymous?: boolean | undefined;
49
49
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
50
+ /**
51
+ * Works via batched iteration of the source archivist to populate the index.
52
+ * @returns A promise that resolves when the background process is complete
53
+ */
50
54
  readonly disallowed?: Record<string, string[]> | undefined;
51
55
  } | undefined;
52
56
  readonly sign?: boolean | undefined;
@@ -64,6 +68,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
64
68
  readonly security?: {
65
69
  readonly allowAnonymous?: boolean | undefined;
66
70
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
71
+ /**
72
+ * Works via batched iteration of the source archivist to populate the index.
73
+ * @returns A promise that resolves when the background process is complete
74
+ */
67
75
  readonly disallowed?: Record<string, string[]> | undefined;
68
76
  } | undefined;
69
77
  readonly sign?: boolean | undefined;
@@ -98,6 +106,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
98
106
  readonly security?: {
99
107
  readonly allowAnonymous?: boolean | undefined;
100
108
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
109
+ /**
110
+ * Works via batched iteration of the source archivist to populate the index.
111
+ * @returns A promise that resolves when the background process is complete
112
+ */
101
113
  readonly disallowed?: Record<string, string[]> | undefined;
102
114
  } | undefined;
103
115
  readonly sign?: boolean | undefined;
@@ -115,6 +127,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
115
127
  readonly security?: {
116
128
  readonly allowAnonymous?: boolean | undefined;
117
129
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
130
+ /**
131
+ * Works via batched iteration of the source archivist to populate the index.
132
+ * @returns A promise that resolves when the background process is complete
133
+ */
118
134
  readonly disallowed?: Record<string, string[]> | undefined;
119
135
  } | undefined;
120
136
  readonly sign?: boolean | undefined;
@@ -202,6 +218,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
202
218
  readonly security?: {
203
219
  readonly allowAnonymous?: boolean | undefined;
204
220
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
221
+ /**
222
+ * Works via batched iteration of the source archivist to populate the index.
223
+ * @returns A promise that resolves when the background process is complete
224
+ */
205
225
  readonly disallowed?: Record<string, string[]> | undefined;
206
226
  } | undefined;
207
227
  readonly sign?: boolean | undefined;
@@ -219,6 +239,10 @@ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = Ind
219
239
  readonly security?: {
220
240
  readonly allowAnonymous?: boolean | undefined;
221
241
  readonly allowed?: Record<string, (string | import("@xyo-network/module-model").CosigningAddressSet)[]> | undefined;
242
+ /**
243
+ * Works via batched iteration of the source archivist to populate the index.
244
+ * @returns A promise that resolves when the background process is complete
245
+ */
222
246
  readonly disallowed?: Record<string, string[]> | undefined;
223
247
  } | undefined;
224
248
  readonly sign?: boolean | undefined;
package/dist/node/Diviner.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAyD,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAEjI,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAE9D,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAsBvF;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBApEvD;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;IAuEH;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBA/EjE;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IAkFH;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;gBAxG5D;;;mBAGG;;;;;;;;;;;;;;;;;;gBAHH;;;mBAGG;;;;;;;;;;;;;;;;;IA2GH;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
package/dist/node/index.cjs CHANGED
@@ -74,8 +74,8 @@ var _IndexingDiviner = class _IndexingDiviner extends import_diviner_abstract.Ab
74
74
  const results = lastState ? await indexCandidateDiviner.divine([
75
75
  lastState
76
76
  ]) : await indexCandidateDiviner.divine();
77
- const nextState = results.find(import_module_model.isModuleState);
78
- const indexCandidates = results.filter((x) => !(0, import_module_model.isModuleState)(x));
77
+ const nextState = results.find(import_module_model.isModuleStateWithMeta);
78
+ const indexCandidates = results.filter((x) => !(0, import_module_model.isModuleStateWithMeta)(x));
79
79
  const toIndexTransformDiviner = await this.getIndexingDivinerStage("indexCandidateToIndexDiviner");
80
80
  const indexes = await toIndexTransformDiviner.divine(indexCandidates);
81
81
  const indexArchivist = await this.getArchivistForStore("indexStore");
@@ -118,7 +118,7 @@ var _IndexingDiviner = class _IndexingDiviner extends import_diviner_abstract.Ab
118
118
  ])));
119
119
  return response.flat();
120
120
  }))).flat();
121
- return results;
121
+ return await Promise.all(results.map((result) => import_payload_builder.PayloadBuilder.build(result)));
122
122
  }
123
123
  /**
124
124
  * Retrieves the archivist for the specified store
package/dist/node/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await (await new BoundWitnessBuilder().payload(nextState)).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,mBAA6C;AAC7C,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,8BAAgC;AAChC,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA+E;AAC/E,6BAA+B;;;;;;;;;;;;AAO/B,IAAMA,aAAa;AAGZ,IAAMC,mBAAN,MAAMA,yBAKHC,wCAAAA;EAIAC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,iCAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,KAACH,mCAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;;AAExE,QAAIA,UAAUY,MAAMC,aAAW,UAAKzB,eAAL,mBAAiBwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,OAAO,MAAM,IAAIC,gDAAAA,EAAsBC,QAAQjB,SAAAA,GAAYkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AACnG,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;;AACvD,UAAMC,WAAOC,yBAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBrB,WAAW,MAAM,GAAG7B,UAAAA,gBAA0BkD,KAAAA,0BAA+B;AACzH,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,eAAoBC,IAAAA,GAAO;AACpH,WAAOI,0CAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;;AACjE,UAAMC,WAAOC,yBAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBQ,qBAAqB,MAAM,GAAG1D,UAAAA,gBAA0BkD,KAAAA,oCAAyC;AAC7I,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,yBAA8BC,IAAAA,GAAO;AAC9H,WAAOQ,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;;AACvE,UAAMC,oBAAgBT,yBACpB,gBAAK9C,WAAL,mBAAawD,0BAAb,mBAAqCF,YACrC,MAAM,GAAG5D,UAAAA,sCAAgD4D,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,eAAOT,4BAASW,wCAAkBV,GAAAA,GAAM,MAAM,GAAGrD,UAAAA,kDAA4D4D,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;;AAC5D,UAAMC,WAAOC,yBAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBc,gBAAgB,MAAM,GAAGhE,UAAAA,gBAA0BkD,KAAAA,+BAAoC;AACnI,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,oBAAyBC,IAAAA,GAAO;AACzH,WAAOQ,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,sCAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,cAAIG,0CAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAAA;;AAAUN,qCAAaJ,oBAAbI,mBAA+BM,6BAAMH,YAAWN,wCAAoBG,aAAaO,eAAeD,6BAAMH,KAAAA,IAASE;WAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,iCAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,cAAU0F,2BAAa,YAAA;AAC1B,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAAS8F,2CAAe,KAAK9F,OAAO;AAC7C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA1MUL;AACR,cANWD,kBAMckG,sBAAqB;AAC9C,cAPWlG,kBAOcmG,iBAA0B;EAACC;EAA6BC;;AAP5E,IAAMrG,kBAAN;AAAMA,kBAAAA,aAAAA;MADZsG,qCAAAA;GACYtG,eAAAA;","names":["moduleName","IndexingDiviner","AbstractDiviner","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","creatableModule"]}
1
+ {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, isModuleStateWithMeta, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleStateWithMeta<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleStateWithMeta(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await (await new BoundWitnessBuilder().payload(nextState)).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<WithMeta<TOut>[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return (await Promise.all(results.map((result) => PayloadBuilder.build(result)))) as WithMeta<TOut>[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload as WithMeta<ModuleState<IndexingDivinerState>>\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,mBAA6C;AAC7C,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,8BAAgC;AAChC,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAAsG;AACtG,6BAA+B;;;;;;;;;;;;AAO/B,IAAMA,aAAa;AAGZ,IAAMC,mBAAN,MAAMA,yBAKHC,wCAAAA;EAIAC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,yCAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,KAACH,2CAAsBG,CAAAA,CAAAA;AAErE,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;;AAExE,QAAIA,UAAUY,MAAMC,aAAW,UAAKzB,eAAL,mBAAiBwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,OAAO,MAAM,IAAIC,gDAAAA,EAAsBC,QAAQjB,SAAAA,GAAYkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AACnG,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAA+B;AACtF,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAQ,MAAMN,QAAQC,IAAI9B,QAAQ+B,IAAI,CAACM,WAAWC,sCAAehB,MAAMe,MAAAA,CAAAA,CAAAA;EACzE;;;;;;EAOA,MAAgB1B,qBAAqB4B,OAAoB;;AACvD,UAAMC,WAAOC,yBAAS,gBAAKhD,WAAL,mBAAc8C,WAAd,mBAAsBvB,WAAW,MAAM,GAAG7B,UAAAA,gBAA0BoD,KAAAA,0BAA+B;AACzH,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGrD,UAAAA,uBAAiCoD,KAAAA,eAAoBC,IAAAA,GAAO;AACpH,WAAOI,0CAAiBC,KAAKH,KAAK,KAAKrB,OAAO;EAChD;;;;;;EAOA,MAAgByB,+BAA+BP,OAAoB;;AACjE,UAAMC,WAAOC,yBAAS,gBAAKhD,WAAL,mBAAc8C,WAAd,mBAAsBQ,qBAAqB,MAAM,GAAG5D,UAAAA,gBAA0BoD,KAAAA,oCAAyC;AAC7I,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGrD,UAAAA,uBAAiCoD,KAAAA,yBAA8BC,IAAAA,GAAO;AAC9H,WAAOQ,sCAAeH,KAAKH,KAAK,KAAKrB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBkD,WAAiC;;AACvE,UAAMC,oBAAgBT,yBACpB,gBAAKhD,WAAL,mBAAa0D,0BAAb,mBAAqCF,YACrC,MAAM,GAAG9D,UAAAA,sCAAgD8D,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,eAAOT,4BAASW,wCAAkBV,GAAAA,GAAM,MAAM,GAAGvD,UAAAA,kDAA4D8D,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBvB,0BAA0Ba,OAAoB;;AAC5D,UAAMC,WAAOC,yBAAS,gBAAKhD,WAAL,mBAAc8C,WAAd,mBAAsBc,gBAAgB,MAAM,GAAGlE,UAAAA,gBAA0BoD,KAAAA,+BAAoC;AACnI,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGrD,UAAAA,uBAAiCoD,KAAAA,oBAAyBC,IAAAA,GAAO;AACzH,WAAOQ,sCAAeH,KAAKH,KAAK,KAAKrB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAIgE,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIlB,sCAAgD;MAAEmB,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKvC,QAAQuC;MACtBC,OAAO;MACP9C,QAAQ;MACR+C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACC1C,MAAK;AACR,UAAM2C,iBAAiB,MAAMV,QAAQtD,OAAO;MAACuD;KAAM;AACnD,QAAIS,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,cAAIG,0CAAeD,YAAAA,GAAe;AAEhCb,eAAOa,aAAaE,UACjBtC,IAAI,CAAC6B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1ChE,OAAO,CAAC,EAAEsD,QAAO,MAAOA,YAAY,KAAKvC,QAAQuC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAAA;;AAAUN,qCAAaJ,oBAAbI,mBAA+BM,6BAAMH,YAAWN,wCAAoBG,aAAaO,eAAeD,6BAAMH,KAAAA,IAASE;WAChI,EAAA;MAEN;IACF;AAGA,QAAIlB,MAAM;AAER,YAAMtC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU2D,IAAI;QAACrB;OAAK,GAAGnD,KAAKyE,iCAAAA;AACnD,UAAIzD,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAO0D;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAK1F,SAAS;AAChB2F,mBAAa,KAAK3F,OAAO;AACzB,WAAKA,UAAUsF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKxF,cAAU4F,2BAAa,YAAA;AAC1B,UAAI;AACF,cAAMtD,QAAQc,QAAO;AACrB,cAAM,KAAKhD,iBAAgB;MAC7B,SAASyF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK7F;AAASgG,2CAAe,KAAKhG,OAAO;AAC7C,aAAKA,UAAUsF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKrF,aAAa;EACvB;AACF;AA1MUL;AACR,cANWD,kBAMcoG,sBAAqB;AAC9C,cAPWpG,kBAOcqG,iBAA0B;EAACC;EAA6BC;;AAP5E,IAAMvG,kBAAN;AAAMA,kBAAAA,aAAAA;MADZwG,qCAAAA;GACYxG,eAAAA;","names":["moduleName","IndexingDiviner","AbstractDiviner","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleStateWithMeta","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","result","PayloadBuilder","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","isModuleState","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","creatableModule"]}
package/dist/node/index.js CHANGED
@@ -17,7 +17,7 @@ import { BoundWitnessDivinerQuerySchema } from "@xyo-network/diviner-boundwitnes
17
17
  import { IndexingDivinerConfigSchema } from "@xyo-network/diviner-indexing-model";
18
18
  import { asDivinerInstance, DivinerConfigSchema } from "@xyo-network/diviner-model";
19
19
  import { DivinerWrapper } from "@xyo-network/diviner-wrapper";
20
- import { creatableModule, isModuleState, ModuleStateSchema } from "@xyo-network/module-model";
20
+ import { creatableModule, isModuleState, isModuleStateWithMeta, ModuleStateSchema } from "@xyo-network/module-model";
21
21
  import { PayloadBuilder } from "@xyo-network/payload-builder";
22
22
  function _ts_decorate(decorators, target, key, desc) {
23
23
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
@@ -50,8 +50,8 @@ var _IndexingDiviner = class _IndexingDiviner extends AbstractDiviner {
50
50
  const results = lastState ? await indexCandidateDiviner.divine([
51
51
  lastState
52
52
  ]) : await indexCandidateDiviner.divine();
53
- const nextState = results.find(isModuleState);
54
- const indexCandidates = results.filter((x) => !isModuleState(x));
53
+ const nextState = results.find(isModuleStateWithMeta);
54
+ const indexCandidates = results.filter((x) => !isModuleStateWithMeta(x));
55
55
  const toIndexTransformDiviner = await this.getIndexingDivinerStage("indexCandidateToIndexDiviner");
56
56
  const indexes = await toIndexTransformDiviner.divine(indexCandidates);
57
57
  const indexArchivist = await this.getArchivistForStore("indexStore");
@@ -94,7 +94,7 @@ var _IndexingDiviner = class _IndexingDiviner extends AbstractDiviner {
94
94
  ])));
95
95
  return response.flat();
96
96
  }))).flat();
97
- return results;
97
+ return await Promise.all(results.map((result) => PayloadBuilder.build(result)));
98
98
  }
99
99
  /**
100
100
  * Retrieves the archivist for the specified store
package/dist/node/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await (await new BoundWitnessBuilder().payload(nextState)).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,gBAAgBC,oBAAoB;AAC7C,SAASC,wBAAwB;AACjC,SAASC,2BAA2B;AACpC,SAASC,sBAAsB;AAC/B,SAASC,uBAAuB;AAChC,SAA0CC,sCAAsC;AAChF,SAEEC,mCAIK;AACP,SAASC,mBAAmBC,2BAAkE;AAC9F,SAASC,sBAAsB;AAC/B,SAASC,iBAAiBC,eAA4BC,yBAAyB;AAC/E,SAASC,sBAAsB;;;;;;;;;;;;AAO/B,IAAMC,aAAa;AAGZ,IAAMC,mBAAN,MAAMA,yBAKHC,gBAAAA;EAIAC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,aAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,CAACH,cAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;;AAExE,QAAIA,UAAUY,MAAMC,aAAW,UAAKzB,eAAL,mBAAiBwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,OAAO,MAAM,IAAIC,oBAAAA,EAAsBC,QAAQjB,SAAAA,GAAYkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AACnG,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;;AACvD,UAAMC,OAAOC,UAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBrB,WAAW,MAAM,GAAG7B,UAAAA,gBAA0BkD,KAAAA,0BAA+B;AACzH,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,eAAoBC,IAAAA,GAAO;AACpH,WAAOI,iBAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;;AACjE,UAAMC,OAAOC,UAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBQ,qBAAqB,MAAM,GAAG1D,UAAAA,gBAA0BkD,KAAAA,oCAAyC;AAC7I,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,yBAA8BC,IAAAA,GAAO;AAC9H,WAAOQ,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;;AACvE,UAAMC,gBAAgBT,UACpB,gBAAK9C,WAAL,mBAAawD,0BAAb,mBAAqCF,YACrC,MAAM,GAAG5D,UAAAA,sCAAgD4D,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,WAAOT,SAASW,kBAAkBV,GAAAA,GAAM,MAAM,GAAGrD,UAAAA,kDAA4D4D,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;;AAC5D,UAAMC,OAAOC,UAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBc,gBAAgB,MAAM,GAAGhE,UAAAA,gBAA0BkD,KAAAA,+BAAoC;AACnI,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,oBAAyBC,IAAAA,GAAO;AACzH,WAAOQ,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,eAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,UAAIG,eAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAAA;;AAAUN,qCAAaJ,oBAAbI,mBAA+BM,6BAAMH,YAAWN,oBAAoBG,aAAaO,eAAeD,6BAAMH,KAAAA,IAASE;WAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,aAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,UAAU0F,aAAa,YAAA;AAC1B,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAAS8F,yBAAe,KAAK9F,OAAO;AAC7C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA1MUL;AACR,cANWD,kBAMckG,sBAAqB;AAC9C,cAPWlG,kBAOcmG,iBAA0B;EAACC;EAA6BC;;AAP5E,IAAMrG,kBAAN;AAAMA,kBAAAA,aAAAA;EADZsG,gBAAAA;GACYtG,eAAAA;","names":["assertEx","clearTimeoutEx","setTimeoutEx","ArchivistWrapper","BoundWitnessBuilder","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","IndexingDivinerConfigSchema","asDivinerInstance","DivinerConfigSchema","DivinerWrapper","creatableModule","isModuleState","ModuleStateSchema","PayloadBuilder","moduleName","IndexingDiviner","AbstractDiviner","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","creatableModule"]}
1
+ {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, isModuleStateWithMeta, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleStateWithMeta<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleStateWithMeta(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await (await new BoundWitnessBuilder().payload(nextState)).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<WithMeta<TOut>[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return (await Promise.all(results.map((result) => PayloadBuilder.build(result)))) as WithMeta<TOut>[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload as WithMeta<ModuleState<IndexingDivinerState>>\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,gBAAgBC,oBAAoB;AAC7C,SAASC,wBAAwB;AACjC,SAASC,2BAA2B;AACpC,SAASC,sBAAsB;AAC/B,SAASC,uBAAuB;AAChC,SAA0CC,sCAAsC;AAChF,SAEEC,mCAIK;AACP,SAASC,mBAAmBC,2BAAkE;AAC9F,SAASC,sBAAsB;AAC/B,SAASC,iBAAiBC,eAAeC,uBAAoCC,yBAAyB;AACtG,SAASC,sBAAsB;;;;;;;;;;;;AAO/B,IAAMC,aAAa;AAGZ,IAAMC,mBAAN,MAAMA,yBAKHC,gBAAAA;EAIAC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,qBAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,CAACH,sBAAsBG,CAAAA,CAAAA;AAErE,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;;AAExE,QAAIA,UAAUY,MAAMC,aAAW,UAAKzB,eAAL,mBAAiBwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,OAAO,MAAM,IAAIC,oBAAAA,EAAsBC,QAAQjB,SAAAA,GAAYkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AACnG,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAA+B;AACtF,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAQ,MAAMN,QAAQC,IAAI9B,QAAQ+B,IAAI,CAACM,WAAWC,eAAehB,MAAMe,MAAAA,CAAAA,CAAAA;EACzE;;;;;;EAOA,MAAgB1B,qBAAqB4B,OAAoB;;AACvD,UAAMC,OAAOC,UAAS,gBAAKhD,WAAL,mBAAc8C,WAAd,mBAAsBvB,WAAW,MAAM,GAAG7B,UAAAA,gBAA0BoD,KAAAA,0BAA+B;AACzH,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGrD,UAAAA,uBAAiCoD,KAAAA,eAAoBC,IAAAA,GAAO;AACpH,WAAOI,iBAAiBC,KAAKH,KAAK,KAAKrB,OAAO;EAChD;;;;;;EAOA,MAAgByB,+BAA+BP,OAAoB;;AACjE,UAAMC,OAAOC,UAAS,gBAAKhD,WAAL,mBAAc8C,WAAd,mBAAsBQ,qBAAqB,MAAM,GAAG5D,UAAAA,gBAA0BoD,KAAAA,oCAAyC;AAC7I,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGrD,UAAAA,uBAAiCoD,KAAAA,yBAA8BC,IAAAA,GAAO;AAC9H,WAAOQ,eAAeH,KAAKH,KAAK,KAAKrB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBkD,WAAiC;;AACvE,UAAMC,gBAAgBT,UACpB,gBAAKhD,WAAL,mBAAa0D,0BAAb,mBAAqCF,YACrC,MAAM,GAAG9D,UAAAA,sCAAgD8D,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,WAAOT,SAASW,kBAAkBV,GAAAA,GAAM,MAAM,GAAGvD,UAAAA,kDAA4D8D,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBvB,0BAA0Ba,OAAoB;;AAC5D,UAAMC,OAAOC,UAAS,gBAAKhD,WAAL,mBAAc8C,WAAd,mBAAsBc,gBAAgB,MAAM,GAAGlE,UAAAA,gBAA0BoD,KAAAA,+BAAoC;AACnI,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGrD,UAAAA,uBAAiCoD,KAAAA,oBAAyBC,IAAAA,GAAO;AACzH,WAAOQ,eAAeH,KAAKH,KAAK,KAAKrB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAIgE,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIlB,eAAgD;MAAEmB,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKvC,QAAQuC;MACtBC,OAAO;MACP9C,QAAQ;MACR+C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACC1C,MAAK;AACR,UAAM2C,iBAAiB,MAAMV,QAAQtD,OAAO;MAACuD;KAAM;AACnD,QAAIS,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,UAAIG,eAAeD,YAAAA,GAAe;AAEhCb,eAAOa,aAAaE,UACjBtC,IAAI,CAAC6B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1ChE,OAAO,CAAC,EAAEsD,QAAO,MAAOA,YAAY,KAAKvC,QAAQuC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAAA;;AAAUN,qCAAaJ,oBAAbI,mBAA+BM,6BAAMH,YAAWN,oBAAoBG,aAAaO,eAAeD,6BAAMH,KAAAA,IAASE;WAChI,EAAA;MAEN;IACF;AAGA,QAAIlB,MAAM;AAER,YAAMtC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU2D,IAAI;QAACrB;OAAK,GAAGnD,KAAKyE,aAAAA;AACnD,UAAIzD,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAO0D;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAK1F,SAAS;AAChB2F,mBAAa,KAAK3F,OAAO;AACzB,WAAKA,UAAUsF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKxF,UAAU4F,aAAa,YAAA;AAC1B,UAAI;AACF,cAAMtD,QAAQc,QAAO;AACrB,cAAM,KAAKhD,iBAAgB;MAC7B,SAASyF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK7F;AAASgG,yBAAe,KAAKhG,OAAO;AAC7C,aAAKA,UAAUsF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKrF,aAAa;EACvB;AACF;AA1MUL;AACR,cANWD,kBAMcoG,sBAAqB;AAC9C,cAPWpG,kBAOcqG,iBAA0B;EAACC;EAA6BC;;AAP5E,IAAMvG,kBAAN;AAAMA,kBAAAA,aAAAA;EADZwG,gBAAAA;GACYxG,eAAAA;","names":["assertEx","clearTimeoutEx","setTimeoutEx","ArchivistWrapper","BoundWitnessBuilder","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","IndexingDivinerConfigSchema","asDivinerInstance","DivinerConfigSchema","DivinerWrapper","creatableModule","isModuleState","isModuleStateWithMeta","ModuleStateSchema","PayloadBuilder","moduleName","IndexingDiviner","AbstractDiviner","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleStateWithMeta","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","result","PayloadBuilder","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","isModuleState","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","creatableModule"]}
package/package.json CHANGED
@@ -10,19 +10,19 @@
10
10
  "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js/issues"
11
11
  },
12
12
  "dependencies": {
13
- "@xylabs/assert": "^2.14.0",
14
- "@xylabs/timer": "^2.14.0",
15
- "@xyo-network/archivist-wrapper": "~2.90.2",
16
- "@xyo-network/boundwitness-builder": "~2.90.2",
17
- "@xyo-network/boundwitness-model": "~2.90.2",
18
- "@xyo-network/diviner-abstract": "~2.90.2",
19
- "@xyo-network/diviner-boundwitness-model": "~2.90.2",
20
- "@xyo-network/diviner-indexing-model": "~2.90.2",
21
- "@xyo-network/diviner-model": "~2.90.2",
22
- "@xyo-network/diviner-wrapper": "~2.90.2",
23
- "@xyo-network/module-model": "~2.90.2",
24
- "@xyo-network/payload-builder": "~2.90.2",
25
- "@xyo-network/payload-model": "~2.90.2"
13
+ "@xylabs/assert": "^2.14.1",
14
+ "@xylabs/timer": "^2.14.1",
15
+ "@xyo-network/archivist-wrapper": "~2.90.3",
16
+ "@xyo-network/boundwitness-builder": "~2.90.3",
17
+ "@xyo-network/boundwitness-model": "~2.90.3",
18
+ "@xyo-network/diviner-abstract": "~2.90.3",
19
+ "@xyo-network/diviner-boundwitness-model": "~2.90.3",
20
+ "@xyo-network/diviner-indexing-model": "~2.90.3",
21
+ "@xyo-network/diviner-model": "~2.90.3",
22
+ "@xyo-network/diviner-wrapper": "~2.90.3",
23
+ "@xyo-network/module-model": "~2.90.3",
24
+ "@xyo-network/payload-builder": "~2.90.3",
25
+ "@xyo-network/payload-model": "~2.90.3"
26
26
  },
27
27
  "devDependencies": {
28
28
  "@xylabs/ts-scripts-yarn3": "^3.2.42",
@@ -68,6 +68,6 @@
68
68
  "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
69
69
  },
70
70
  "sideEffects": false,
71
- "version": "2.90.2",
71
+ "version": "2.90.3",
72
72
  "type": "module"
73
73
  }
package/src/Diviner.ts CHANGED
@@ -14,9 +14,9 @@ import {
14
14
  } from '@xyo-network/diviner-indexing-model'
15
15
  import { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'
16
16
  import { DivinerWrapper } from '@xyo-network/diviner-wrapper'
17
- import { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'
17
+ import { creatableModule, isModuleState, isModuleStateWithMeta, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'
18
18
  import { PayloadBuilder } from '@xyo-network/payload-builder'
19
- import { Payload } from '@xyo-network/payload-model'
19
+ import { Payload, WithMeta } from '@xyo-network/payload-model'
20
20
 
21
21
  export type ConfigStoreKey = 'indexStore' | 'stateStore'
22
22
 
@@ -56,8 +56,8 @@ export class IndexingDiviner<
56
56
  const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')
57
57
  const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()
58
58
  // Filter next state out from results
59
- const nextState = results.find(isModuleState<IndexingDivinerState>)
60
- const indexCandidates = results.filter((x) => !isModuleState(x))
59
+ const nextState = results.find(isModuleStateWithMeta<IndexingDivinerState>)
60
+ const indexCandidates = results.filter((x) => !isModuleStateWithMeta(x))
61
61
  // Transform candidates to indexes
62
62
  const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')
63
63
  const indexes = await toIndexTransformDiviner.divine(indexCandidates)
@@ -86,7 +86,7 @@ export class IndexingDiviner<
86
86
  await archivist.insert([bw, nextState])
87
87
  }
88
88
 
89
- protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {
89
+ protected override async divineHandler(payloads: TIn[] = []): Promise<WithMeta<TOut>[]> {
90
90
  const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')
91
91
  const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')
92
92
  const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')
@@ -105,7 +105,7 @@ export class IndexingDiviner<
105
105
  )
106
106
  ).flat()
107
107
  // TODO: Infer this type over casting to this type
108
- return results as TOut[]
108
+ return (await Promise.all(results.map((result) => PayloadBuilder.build(result)))) as WithMeta<TOut>[]
109
109
  }
110
110
 
111
111
  /**
@@ -194,7 +194,7 @@ export class IndexingDiviner<
194
194
  const archivist = await this.getArchivistForStore('stateStore')
195
195
  const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)
196
196
  if (payload) {
197
- return payload
197
+ return payload as WithMeta<ModuleState<IndexingDivinerState>>
198
198
  }
199
199
  }
200
200
  return undefined