@xyo-network/diviner-indexing-memory 2.84.19 → 2.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/Diviner.d.cts +1 -0
- package/dist/browser/Diviner.d.cts.map +1 -1
- package/dist/browser/Diviner.d.mts +1 -0
- package/dist/browser/Diviner.d.mts.map +1 -1
- package/dist/browser/Diviner.d.ts +1 -0
- package/dist/browser/Diviner.d.ts.map +1 -1
- package/dist/browser/index.cjs +89 -56
- package/dist/browser/index.cjs.map +1 -1
- package/dist/browser/index.js +93 -60
- package/dist/browser/index.js.map +1 -1
- package/dist/node/Diviner.d.cts +1 -0
- package/dist/node/Diviner.d.cts.map +1 -1
- package/dist/node/Diviner.d.mts +1 -0
- package/dist/node/Diviner.d.mts.map +1 -1
- package/dist/node/Diviner.d.ts +1 -0
- package/dist/node/Diviner.d.ts.map +1 -1
- package/dist/node/index.cjs +97 -60
- package/dist/node/index.cjs.map +1 -1
- package/dist/node/index.js +101 -64
- package/dist/node/index.js.map +1 -1
- package/package.json +12 -12
- package/src/Diviner.ts +3 -1
|
@@ -8,6 +8,7 @@ import { Payload } from '@xyo-network/payload-model';
|
|
|
8
8
|
export type ConfigStoreKey = 'indexStore' | 'stateStore';
|
|
9
9
|
export type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>;
|
|
10
10
|
export declare class IndexingDiviner<TParams extends IndexingDivinerParams = IndexingDivinerParams, TIn extends Payload = Payload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {
|
|
11
|
+
static readonly allowRandomAccount = false;
|
|
11
12
|
static readonly configSchemas: string[];
|
|
12
13
|
private _lastState?;
|
|
13
14
|
private _pollId?;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAIjE,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,
|
|
1
|
+
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAIjE,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
|
|
@@ -8,6 +8,7 @@ import { Payload } from '@xyo-network/payload-model';
|
|
|
8
8
|
export type ConfigStoreKey = 'indexStore' | 'stateStore';
|
|
9
9
|
export type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>;
|
|
10
10
|
export declare class IndexingDiviner<TParams extends IndexingDivinerParams = IndexingDivinerParams, TIn extends Payload = Payload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {
|
|
11
|
+
static readonly allowRandomAccount = false;
|
|
11
12
|
static readonly configSchemas: string[];
|
|
12
13
|
private _lastState?;
|
|
13
14
|
private _pollId?;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAIjE,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,
|
|
1
|
+
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAIjE,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
|
|
@@ -8,6 +8,7 @@ import { Payload } from '@xyo-network/payload-model';
|
|
|
8
8
|
export type ConfigStoreKey = 'indexStore' | 'stateStore';
|
|
9
9
|
export type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>;
|
|
10
10
|
export declare class IndexingDiviner<TParams extends IndexingDivinerParams = IndexingDivinerParams, TIn extends Payload = Payload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {
|
|
11
|
+
static readonly allowRandomAccount = false;
|
|
11
12
|
static readonly configSchemas: string[];
|
|
12
13
|
private _lastState?;
|
|
13
14
|
private _pollId?;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAIjE,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,
|
|
1
|
+
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAIjE,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
|
package/dist/browser/index.cjs
CHANGED
|
@@ -3,6 +3,7 @@ var __defProp = Object.defineProperty;
|
|
|
3
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
6
7
|
var __export = (target, all) => {
|
|
7
8
|
for (var name in all)
|
|
8
9
|
__defProp(target, name, { get: all[name], enumerable: true });
|
|
@@ -36,9 +37,27 @@ var import_diviner_model = require("@xyo-network/diviner-model");
|
|
|
36
37
|
var import_diviner_wrapper = require("@xyo-network/diviner-wrapper");
|
|
37
38
|
var import_module_model = require("@xyo-network/module-model");
|
|
38
39
|
var import_payload_builder = require("@xyo-network/payload-builder");
|
|
40
|
+
function _ts_decorate(decorators, target, key, desc) {
|
|
41
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
42
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
43
|
+
r = Reflect.decorate(decorators, target, key, desc);
|
|
44
|
+
else
|
|
45
|
+
for (var i = decorators.length - 1; i >= 0; i--)
|
|
46
|
+
if (d = decorators[i])
|
|
47
|
+
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
48
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
49
|
+
}
|
|
50
|
+
__name(_ts_decorate, "_ts_decorate");
|
|
39
51
|
var moduleName = "IndexingDiviner";
|
|
40
52
|
var IndexingDiviner = class extends import_abstract_diviner.AbstractDiviner {
|
|
41
|
-
static
|
|
53
|
+
static {
|
|
54
|
+
__name(this, "IndexingDiviner");
|
|
55
|
+
}
|
|
56
|
+
static allowRandomAccount = false;
|
|
57
|
+
static configSchemas = [
|
|
58
|
+
import_diviner_indexing_model.IndexingDivinerConfigSchema,
|
|
59
|
+
import_diviner_model.DivinerConfigSchema
|
|
60
|
+
];
|
|
42
61
|
_lastState;
|
|
43
62
|
_pollId;
|
|
44
63
|
get payloadDivinerLimit() {
|
|
@@ -48,13 +67,15 @@ var IndexingDiviner = class extends import_abstract_diviner.AbstractDiviner {
|
|
|
48
67
|
return this.config.pollFrequency ?? 1e4;
|
|
49
68
|
}
|
|
50
69
|
/**
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
70
|
+
* Works via batched iteration of the source archivist to populate the index.
|
|
71
|
+
* @returns A promise that resolves when the background process is complete
|
|
72
|
+
*/
|
|
54
73
|
backgroundDivine = async () => {
|
|
55
74
|
const lastState = await this.retrieveState();
|
|
56
75
|
const indexCandidateDiviner = await this.getIndexingDivinerStage("stateToIndexCandidateDiviner");
|
|
57
|
-
const results = lastState ? await indexCandidateDiviner.divine([
|
|
76
|
+
const results = lastState ? await indexCandidateDiviner.divine([
|
|
77
|
+
lastState
|
|
78
|
+
]) : await indexCandidateDiviner.divine();
|
|
58
79
|
const nextState = results.find(import_module_model.isModuleState);
|
|
59
80
|
const indexCandidates = results.filter((x) => !(0, import_module_model.isModuleState)(x));
|
|
60
81
|
const toIndexTransformDiviner = await this.getIndexingDivinerStage("indexCandidateToIndexDiviner");
|
|
@@ -66,108 +87,117 @@ var IndexingDiviner = class extends import_abstract_diviner.AbstractDiviner {
|
|
|
66
87
|
}
|
|
67
88
|
};
|
|
68
89
|
/**
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
90
|
+
* Commit the internal state of the Diviner process. This is similar
|
|
91
|
+
* to a transaction completion in a database and should only be called
|
|
92
|
+
* when results have been successfully persisted to the appropriate
|
|
93
|
+
* external stores.
|
|
94
|
+
* @param nextState The state to commit
|
|
95
|
+
*/
|
|
75
96
|
async commitState(nextState) {
|
|
76
97
|
if (nextState.state.offset === this._lastState?.state.offset)
|
|
77
98
|
return;
|
|
78
99
|
this._lastState = nextState;
|
|
79
100
|
const archivist = await this.getArchivistForStore("stateStore");
|
|
80
101
|
const [bw] = await new import_boundwitness_builder.BoundWitnessBuilder().payload(nextState).witness(this.account).build();
|
|
81
|
-
await archivist.insert([
|
|
102
|
+
await archivist.insert([
|
|
103
|
+
bw,
|
|
104
|
+
nextState
|
|
105
|
+
]);
|
|
82
106
|
}
|
|
83
107
|
async divineHandler(payloads = []) {
|
|
84
108
|
const indexPayloadDiviner = await this.getPayloadDivinerForStore("indexStore");
|
|
85
109
|
const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage("divinerQueryToIndexQueryDiviner");
|
|
86
110
|
const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage("indexQueryResponseToDivinerQueryResponseDiviner");
|
|
87
|
-
const results = (await Promise.all(
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
111
|
+
const results = (await Promise.all(payloads.map(async (payload) => {
|
|
112
|
+
const indexQuery = await divinerQueryToIndexQueryDiviner.divine([
|
|
113
|
+
payload
|
|
114
|
+
]);
|
|
115
|
+
const indexedResults = await indexPayloadDiviner.divine(indexQuery);
|
|
116
|
+
const response = await Promise.all(indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([
|
|
117
|
+
payload,
|
|
118
|
+
indexedResult
|
|
119
|
+
])));
|
|
120
|
+
return response.flat();
|
|
121
|
+
}))).flat();
|
|
97
122
|
return results;
|
|
98
123
|
}
|
|
99
124
|
/**
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
125
|
+
* Retrieves the archivist for the specified store
|
|
126
|
+
* @param store The store to retrieve the archivist for
|
|
127
|
+
* @returns The archivist for the specified store
|
|
128
|
+
*/
|
|
104
129
|
async getArchivistForStore(store) {
|
|
105
130
|
const name = (0, import_assert.assertEx)(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`);
|
|
106
131
|
const mod = (0, import_assert.assertEx)(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`);
|
|
107
132
|
return import_archivist_wrapper.ArchivistWrapper.wrap(mod, this.account);
|
|
108
133
|
}
|
|
109
134
|
/**
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
135
|
+
* Retrieves the BoundWitness Diviner for the specified store
|
|
136
|
+
* @param store The store to retrieve the BoundWitness Diviner for
|
|
137
|
+
* @returns The BoundWitness Diviner for the specified store
|
|
138
|
+
*/
|
|
114
139
|
async getBoundWitnessDivinerForStore(store) {
|
|
115
140
|
const name = (0, import_assert.assertEx)(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`);
|
|
116
141
|
const mod = (0, import_assert.assertEx)(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`);
|
|
117
142
|
return import_diviner_wrapper.DivinerWrapper.wrap(mod, this.account);
|
|
118
143
|
}
|
|
119
144
|
/**
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
145
|
+
* Gets the Diviner for the supplied Indexing Diviner stage
|
|
146
|
+
* @param transform The Indexing Diviner stage
|
|
147
|
+
* @returns The diviner corresponding to the Indexing Diviner stage
|
|
148
|
+
*/
|
|
124
149
|
async getIndexingDivinerStage(transform) {
|
|
125
|
-
const nameOrAddress = (0, import_assert.assertEx)(
|
|
126
|
-
this.config?.indexingDivinerStages?.[transform],
|
|
127
|
-
() => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`
|
|
128
|
-
);
|
|
150
|
+
const nameOrAddress = (0, import_assert.assertEx)(this.config?.indexingDivinerStages?.[transform], () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`);
|
|
129
151
|
const mod = await this.resolve(nameOrAddress);
|
|
130
152
|
return (0, import_assert.assertEx)((0, import_diviner_model.asDivinerInstance)(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`);
|
|
131
153
|
}
|
|
132
154
|
/**
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
155
|
+
* Retrieves the Payload Diviner for the specified store
|
|
156
|
+
* @param store The store to retrieve the Payload Diviner for
|
|
157
|
+
* @returns The Payload Diviner for the specified store
|
|
158
|
+
*/
|
|
137
159
|
async getPayloadDivinerForStore(store) {
|
|
138
160
|
const name = (0, import_assert.assertEx)(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`);
|
|
139
161
|
const mod = (0, import_assert.assertEx)(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`);
|
|
140
162
|
return import_diviner_wrapper.DivinerWrapper.wrap(mod, this.account);
|
|
141
163
|
}
|
|
142
164
|
/**
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
165
|
+
* Retrieves the last state of the Diviner process. Used to recover state after
|
|
166
|
+
* preemptions, reboots, etc.
|
|
167
|
+
*/
|
|
146
168
|
async retrieveState() {
|
|
147
169
|
if (this._lastState)
|
|
148
170
|
return this._lastState;
|
|
149
171
|
let hash = "";
|
|
150
172
|
const diviner = await this.getBoundWitnessDivinerForStore("stateStore");
|
|
151
|
-
const query = await new import_payload_builder.PayloadBuilder({
|
|
173
|
+
const query = await new import_payload_builder.PayloadBuilder({
|
|
174
|
+
schema: import_diviner_boundwitness_model.BoundWitnessDivinerQuerySchema
|
|
175
|
+
}).fields({
|
|
152
176
|
address: this.account.address,
|
|
153
177
|
limit: 1,
|
|
154
178
|
offset: 0,
|
|
155
179
|
order: "desc",
|
|
156
|
-
payload_schemas: [
|
|
180
|
+
payload_schemas: [
|
|
181
|
+
import_module_model.ModuleStateSchema
|
|
182
|
+
]
|
|
157
183
|
}).build();
|
|
158
|
-
const boundWitnesses = await diviner.divine([
|
|
184
|
+
const boundWitnesses = await diviner.divine([
|
|
185
|
+
query
|
|
186
|
+
]);
|
|
159
187
|
if (boundWitnesses.length > 0) {
|
|
160
188
|
const boundWitness = boundWitnesses[0];
|
|
161
189
|
if ((0, import_boundwitness_model.isBoundWitness)(boundWitness)) {
|
|
162
|
-
hash = boundWitness.addresses.map((address, index) => ({
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
);
|
|
190
|
+
hash = boundWitness.addresses.map((address, index) => ({
|
|
191
|
+
address,
|
|
192
|
+
index
|
|
193
|
+
})).filter(({ address }) => address === this.account.address).reduce((prev, curr) => boundWitness.payload_schemas?.[curr?.index] === import_module_model.ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev, "");
|
|
166
194
|
}
|
|
167
195
|
}
|
|
168
196
|
if (hash) {
|
|
169
197
|
const archivist = await this.getArchivistForStore("stateStore");
|
|
170
|
-
const payload = (await archivist.get([
|
|
198
|
+
const payload = (await archivist.get([
|
|
199
|
+
hash
|
|
200
|
+
])).find(import_module_model.isModuleState);
|
|
171
201
|
if (payload) {
|
|
172
202
|
return payload;
|
|
173
203
|
}
|
|
@@ -187,9 +217,9 @@ var IndexingDiviner = class extends import_abstract_diviner.AbstractDiviner {
|
|
|
187
217
|
return await super.stopHandler();
|
|
188
218
|
}
|
|
189
219
|
/**
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
220
|
+
* Runs the background divine process on a loop with a delay
|
|
221
|
+
* specified by the `config.pollFrequency`
|
|
222
|
+
*/
|
|
193
223
|
poll() {
|
|
194
224
|
this._pollId = setTimeout(async () => {
|
|
195
225
|
try {
|
|
@@ -206,4 +236,7 @@ var IndexingDiviner = class extends import_abstract_diviner.AbstractDiviner {
|
|
|
206
236
|
}, this.pollFrequency);
|
|
207
237
|
}
|
|
208
238
|
};
|
|
239
|
+
IndexingDiviner = _ts_decorate([
|
|
240
|
+
(0, import_module_model.creatableModule)()
|
|
241
|
+
], IndexingDiviner);
|
|
209
242
|
//# sourceMappingURL=index.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA8D;AAC9D,6BAA+B;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,wCAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,2DAA6B,wCAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,iCAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,KAAC,mCAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AAExE,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AACvE,UAAM,oBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,eAAO,4BAAS,wCAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,MAAM,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EAC/G,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EAExD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,wCAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA+E;AAC/E,6BAA+B;;;;;;;;;;;;AAO/B,IAAMA,aAAa;AAGZ,IAAMC,kBAAN,cAKGC,wCAAAA;SAAAA;;;EACR,OAAyBC,qBAAqB;EAC9C,OAAyBC,gBAA0B;IAACC;IAA6BC;;EAEzEC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,iCAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,KAACH,mCAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;AAExE,QAAIA,UAAUY,MAAMC,WAAW,KAAKzB,YAAYwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,MAAM,IAAIC,gDAAAA,EAAsBC,QAAQjB,SAAAA,EAAWkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AAC3F,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;AACvD,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQrB,WAAW,MAAM,GAAGjC,UAAAA,gBAA0BsD,KAAAA,0BAA+B;AACzH,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,YAAiB;AAC1G,WAAOK,0CAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;AACjE,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQQ,qBAAqB,MAAM,GAAG9D,UAAAA,gBAA0BsD,KAAAA,oCAAyC;AAC7I,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,sBAA2B;AACpH,WAAOS,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;AACvE,UAAMC,oBAAgBT,wBACpB,KAAK9C,QAAQwD,wBAAwBF,SAAAA,GACrC,MAAM,GAAGhE,UAAAA,sCAAgDgE,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,eAAOT,4BAASW,wCAAkBV,GAAAA,GAAM,MAAM,GAAGzD,UAAAA,kDAA4DgE,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;AAC5D,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQc,gBAAgB,MAAM,GAAGpE,UAAAA,gBAA0BsD,KAAAA,+BAAoC;AACnI,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,iBAAsB;AAC/G,WAAOS,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,sCAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,cAAIG,0CAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAUN,aAAaJ,kBAAkBU,MAAMH,KAAAA,MAAWN,wCAAoBG,aAAaO,eAAeD,MAAMH,KAAAA,IAASE,MAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,iCAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,UAAU0F,WAAW,YAAA;AACxB,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAASyF,uBAAa,KAAKzF,OAAO;AAC3C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA/MaV,kBAAAA,aAAAA;MADZqG,qCAAAA;GACYrG,eAAAA;","names":["moduleName","IndexingDiviner","AbstractDiviner","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeout","e","console","log","creatableModule"]}
|
package/dist/browser/index.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
3
|
+
|
|
1
4
|
// src/Diviner.ts
|
|
2
5
|
import { assertEx } from "@xylabs/assert";
|
|
3
6
|
import { AbstractDiviner } from "@xyo-network/abstract-diviner";
|
|
@@ -5,16 +8,32 @@ import { ArchivistWrapper } from "@xyo-network/archivist-wrapper";
|
|
|
5
8
|
import { BoundWitnessBuilder } from "@xyo-network/boundwitness-builder";
|
|
6
9
|
import { isBoundWitness } from "@xyo-network/boundwitness-model";
|
|
7
10
|
import { BoundWitnessDivinerQuerySchema } from "@xyo-network/diviner-boundwitness-model";
|
|
8
|
-
import {
|
|
9
|
-
IndexingDivinerConfigSchema
|
|
10
|
-
} from "@xyo-network/diviner-indexing-model";
|
|
11
|
+
import { IndexingDivinerConfigSchema } from "@xyo-network/diviner-indexing-model";
|
|
11
12
|
import { asDivinerInstance, DivinerConfigSchema } from "@xyo-network/diviner-model";
|
|
12
13
|
import { DivinerWrapper } from "@xyo-network/diviner-wrapper";
|
|
13
|
-
import { isModuleState, ModuleStateSchema } from "@xyo-network/module-model";
|
|
14
|
+
import { creatableModule, isModuleState, ModuleStateSchema } from "@xyo-network/module-model";
|
|
14
15
|
import { PayloadBuilder } from "@xyo-network/payload-builder";
|
|
16
|
+
function _ts_decorate(decorators, target, key, desc) {
|
|
17
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
18
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
19
|
+
r = Reflect.decorate(decorators, target, key, desc);
|
|
20
|
+
else
|
|
21
|
+
for (var i = decorators.length - 1; i >= 0; i--)
|
|
22
|
+
if (d = decorators[i])
|
|
23
|
+
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
24
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
25
|
+
}
|
|
26
|
+
__name(_ts_decorate, "_ts_decorate");
|
|
15
27
|
var moduleName = "IndexingDiviner";
|
|
16
28
|
var IndexingDiviner = class extends AbstractDiviner {
|
|
17
|
-
static
|
|
29
|
+
static {
|
|
30
|
+
__name(this, "IndexingDiviner");
|
|
31
|
+
}
|
|
32
|
+
static allowRandomAccount = false;
|
|
33
|
+
static configSchemas = [
|
|
34
|
+
IndexingDivinerConfigSchema,
|
|
35
|
+
DivinerConfigSchema
|
|
36
|
+
];
|
|
18
37
|
_lastState;
|
|
19
38
|
_pollId;
|
|
20
39
|
get payloadDivinerLimit() {
|
|
@@ -24,13 +43,15 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
24
43
|
return this.config.pollFrequency ?? 1e4;
|
|
25
44
|
}
|
|
26
45
|
/**
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
46
|
+
* Works via batched iteration of the source archivist to populate the index.
|
|
47
|
+
* @returns A promise that resolves when the background process is complete
|
|
48
|
+
*/
|
|
30
49
|
backgroundDivine = async () => {
|
|
31
50
|
const lastState = await this.retrieveState();
|
|
32
51
|
const indexCandidateDiviner = await this.getIndexingDivinerStage("stateToIndexCandidateDiviner");
|
|
33
|
-
const results = lastState ? await indexCandidateDiviner.divine([
|
|
52
|
+
const results = lastState ? await indexCandidateDiviner.divine([
|
|
53
|
+
lastState
|
|
54
|
+
]) : await indexCandidateDiviner.divine();
|
|
34
55
|
const nextState = results.find(isModuleState);
|
|
35
56
|
const indexCandidates = results.filter((x) => !isModuleState(x));
|
|
36
57
|
const toIndexTransformDiviner = await this.getIndexingDivinerStage("indexCandidateToIndexDiviner");
|
|
@@ -42,108 +63,117 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
42
63
|
}
|
|
43
64
|
};
|
|
44
65
|
/**
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
66
|
+
* Commit the internal state of the Diviner process. This is similar
|
|
67
|
+
* to a transaction completion in a database and should only be called
|
|
68
|
+
* when results have been successfully persisted to the appropriate
|
|
69
|
+
* external stores.
|
|
70
|
+
* @param nextState The state to commit
|
|
71
|
+
*/
|
|
51
72
|
async commitState(nextState) {
|
|
52
73
|
if (nextState.state.offset === this._lastState?.state.offset)
|
|
53
74
|
return;
|
|
54
75
|
this._lastState = nextState;
|
|
55
76
|
const archivist = await this.getArchivistForStore("stateStore");
|
|
56
77
|
const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build();
|
|
57
|
-
await archivist.insert([
|
|
78
|
+
await archivist.insert([
|
|
79
|
+
bw,
|
|
80
|
+
nextState
|
|
81
|
+
]);
|
|
58
82
|
}
|
|
59
83
|
async divineHandler(payloads = []) {
|
|
60
84
|
const indexPayloadDiviner = await this.getPayloadDivinerForStore("indexStore");
|
|
61
85
|
const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage("divinerQueryToIndexQueryDiviner");
|
|
62
86
|
const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage("indexQueryResponseToDivinerQueryResponseDiviner");
|
|
63
|
-
const results = (await Promise.all(
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
87
|
+
const results = (await Promise.all(payloads.map(async (payload) => {
|
|
88
|
+
const indexQuery = await divinerQueryToIndexQueryDiviner.divine([
|
|
89
|
+
payload
|
|
90
|
+
]);
|
|
91
|
+
const indexedResults = await indexPayloadDiviner.divine(indexQuery);
|
|
92
|
+
const response = await Promise.all(indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([
|
|
93
|
+
payload,
|
|
94
|
+
indexedResult
|
|
95
|
+
])));
|
|
96
|
+
return response.flat();
|
|
97
|
+
}))).flat();
|
|
73
98
|
return results;
|
|
74
99
|
}
|
|
75
100
|
/**
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
101
|
+
* Retrieves the archivist for the specified store
|
|
102
|
+
* @param store The store to retrieve the archivist for
|
|
103
|
+
* @returns The archivist for the specified store
|
|
104
|
+
*/
|
|
80
105
|
async getArchivistForStore(store) {
|
|
81
106
|
const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`);
|
|
82
107
|
const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`);
|
|
83
108
|
return ArchivistWrapper.wrap(mod, this.account);
|
|
84
109
|
}
|
|
85
110
|
/**
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
111
|
+
* Retrieves the BoundWitness Diviner for the specified store
|
|
112
|
+
* @param store The store to retrieve the BoundWitness Diviner for
|
|
113
|
+
* @returns The BoundWitness Diviner for the specified store
|
|
114
|
+
*/
|
|
90
115
|
async getBoundWitnessDivinerForStore(store) {
|
|
91
116
|
const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`);
|
|
92
117
|
const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`);
|
|
93
118
|
return DivinerWrapper.wrap(mod, this.account);
|
|
94
119
|
}
|
|
95
120
|
/**
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
121
|
+
* Gets the Diviner for the supplied Indexing Diviner stage
|
|
122
|
+
* @param transform The Indexing Diviner stage
|
|
123
|
+
* @returns The diviner corresponding to the Indexing Diviner stage
|
|
124
|
+
*/
|
|
100
125
|
async getIndexingDivinerStage(transform) {
|
|
101
|
-
const nameOrAddress = assertEx(
|
|
102
|
-
this.config?.indexingDivinerStages?.[transform],
|
|
103
|
-
() => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`
|
|
104
|
-
);
|
|
126
|
+
const nameOrAddress = assertEx(this.config?.indexingDivinerStages?.[transform], () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`);
|
|
105
127
|
const mod = await this.resolve(nameOrAddress);
|
|
106
128
|
return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`);
|
|
107
129
|
}
|
|
108
130
|
/**
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
131
|
+
* Retrieves the Payload Diviner for the specified store
|
|
132
|
+
* @param store The store to retrieve the Payload Diviner for
|
|
133
|
+
* @returns The Payload Diviner for the specified store
|
|
134
|
+
*/
|
|
113
135
|
async getPayloadDivinerForStore(store) {
|
|
114
136
|
const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`);
|
|
115
137
|
const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`);
|
|
116
138
|
return DivinerWrapper.wrap(mod, this.account);
|
|
117
139
|
}
|
|
118
140
|
/**
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
141
|
+
* Retrieves the last state of the Diviner process. Used to recover state after
|
|
142
|
+
* preemptions, reboots, etc.
|
|
143
|
+
*/
|
|
122
144
|
async retrieveState() {
|
|
123
145
|
if (this._lastState)
|
|
124
146
|
return this._lastState;
|
|
125
147
|
let hash = "";
|
|
126
148
|
const diviner = await this.getBoundWitnessDivinerForStore("stateStore");
|
|
127
|
-
const query = await new PayloadBuilder({
|
|
149
|
+
const query = await new PayloadBuilder({
|
|
150
|
+
schema: BoundWitnessDivinerQuerySchema
|
|
151
|
+
}).fields({
|
|
128
152
|
address: this.account.address,
|
|
129
153
|
limit: 1,
|
|
130
154
|
offset: 0,
|
|
131
155
|
order: "desc",
|
|
132
|
-
payload_schemas: [
|
|
156
|
+
payload_schemas: [
|
|
157
|
+
ModuleStateSchema
|
|
158
|
+
]
|
|
133
159
|
}).build();
|
|
134
|
-
const boundWitnesses = await diviner.divine([
|
|
160
|
+
const boundWitnesses = await diviner.divine([
|
|
161
|
+
query
|
|
162
|
+
]);
|
|
135
163
|
if (boundWitnesses.length > 0) {
|
|
136
164
|
const boundWitness = boundWitnesses[0];
|
|
137
165
|
if (isBoundWitness(boundWitness)) {
|
|
138
|
-
hash = boundWitness.addresses.map((address, index) => ({
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
);
|
|
166
|
+
hash = boundWitness.addresses.map((address, index) => ({
|
|
167
|
+
address,
|
|
168
|
+
index
|
|
169
|
+
})).filter(({ address }) => address === this.account.address).reduce((prev, curr) => boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev, "");
|
|
142
170
|
}
|
|
143
171
|
}
|
|
144
172
|
if (hash) {
|
|
145
173
|
const archivist = await this.getArchivistForStore("stateStore");
|
|
146
|
-
const payload = (await archivist.get([
|
|
174
|
+
const payload = (await archivist.get([
|
|
175
|
+
hash
|
|
176
|
+
])).find(isModuleState);
|
|
147
177
|
if (payload) {
|
|
148
178
|
return payload;
|
|
149
179
|
}
|
|
@@ -163,9 +193,9 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
163
193
|
return await super.stopHandler();
|
|
164
194
|
}
|
|
165
195
|
/**
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
196
|
+
* Runs the background divine process on a loop with a delay
|
|
197
|
+
* specified by the `config.pollFrequency`
|
|
198
|
+
*/
|
|
169
199
|
poll() {
|
|
170
200
|
this._pollId = setTimeout(async () => {
|
|
171
201
|
try {
|
|
@@ -182,6 +212,9 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
182
212
|
}, this.pollFrequency);
|
|
183
213
|
}
|
|
184
214
|
};
|
|
215
|
+
IndexingDiviner = _ts_decorate([
|
|
216
|
+
creatableModule()
|
|
217
|
+
], IndexingDiviner);
|
|
185
218
|
export {
|
|
186
219
|
IndexingDiviner
|
|
187
220
|
};
|