@xyo-network/diviner-indexing 7.2.3 → 7.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,3 @@
1
- export * from '@xyo-network/diviner-indexing-memory';
2
- export * from '@xyo-network/diviner-indexing-model';
3
- export * from '@xyo-network/diviner-temporal-indexing';
1
+ export * from './memory.ts';
2
+ export * from './model.ts';
4
3
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,sCAAsC,CAAA;AACpD,cAAc,qCAAqC,CAAA;AACnD,cAAc,wCAAwC,CAAA"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAA;AAC3B,cAAc,YAAY,CAAA"}
@@ -1,5 +1,258 @@
1
- // src/index.ts
2
- export * from "@xyo-network/diviner-indexing-memory";
3
- export * from "@xyo-network/diviner-indexing-model";
4
- export * from "@xyo-network/diviner-temporal-indexing";
1
+ var __defProp = Object.defineProperty;
2
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
3
+ var __getProtoOf = Object.getPrototypeOf;
4
+ var __reflectGet = Reflect.get;
5
+ var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
6
+ var __decorateClass = (decorators, target, key, kind) => {
7
+ var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
8
+ for (var i = decorators.length - 1, decorator; i >= 0; i--)
9
+ if (decorator = decorators[i])
10
+ result = (kind ? decorator(target, key, result) : decorator(result)) || result;
11
+ if (kind && result) __defProp(target, key, result);
12
+ return result;
13
+ };
14
+ var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
15
+ var __superGet = (cls, obj, key) => __reflectGet(__getProtoOf(cls), key, obj);
16
+
17
+ // src/memory/Diviner.ts
18
+ import {
19
+ assertEx,
20
+ clearTimeoutEx,
21
+ isString,
22
+ setTimeoutEx
23
+ } from "@ariestools/sdk";
24
+ import { ArchivistWrapper } from "@xyo-network/sdk/archivist-wrapper";
25
+ import { AbstractDiviner } from "@xyo-network/sdk/diviner-abstract";
26
+ import { BoundWitnessDivinerQuerySchema } from "@xyo-network/sdk/diviner-boundwitness/model";
27
+ import { asDivinerInstance } from "@xyo-network/sdk/diviner-model";
28
+ import { DivinerWrapper } from "@xyo-network/sdk/diviner-wrapper";
29
+ import {
30
+ creatableModule,
31
+ isModuleState,
32
+ ModuleStateSchema
33
+ } from "@xyo-network/sdk/module-model";
34
+ import {
35
+ BoundWitnessBuilder,
36
+ isBoundWitness,
37
+ PayloadBuilder,
38
+ SequenceConstants
39
+ } from "@xyo-network/sdk-protocol";
40
+
41
+ // src/model/Config.ts
42
+ import {
43
+ zodAsFactory,
44
+ zodIsFactory,
45
+ zodToFactory
46
+ } from "@ariestools/sdk";
47
+ import { DivinerConfigZod } from "@xyo-network/sdk/diviner-model";
48
+ import {
49
+ asSchema as asSchema2
50
+ } from "@xyo-network/sdk-protocol";
51
+ import * as z from "zod/mini";
52
+
53
+ // src/model/Schema.ts
54
+ import { asSchema } from "@xyo-network/sdk-protocol";
55
+ var IndexingDivinerSchema = asSchema("network.xyo.diviner.indexing", true);
56
+
57
+ // src/model/Config.ts
58
+ var IndexingDivinerConfigSchema = asSchema2(`${IndexingDivinerSchema}.config`, true);
59
+ var IndexingDivinerConfigZod = z.extend(DivinerConfigZod, {
60
+ indexStore: z.optional(z.custom()),
61
+ indexingDivinerStages: z.optional(z.custom()),
62
+ payloadDivinerLimit: z.optional(z.number()),
63
+ pollFrequency: z.optional(z.number()),
64
+ schema: z.literal(IndexingDivinerConfigSchema),
65
+ stateStore: z.optional(z.custom())
66
+ });
67
+ var isIndexingDivinerConfig = zodIsFactory(IndexingDivinerConfigZod);
68
+ var asIndexingDivinerConfig = zodAsFactory(IndexingDivinerConfigZod, "asIndexingDivinerConfig");
69
+ var toIndexingDivinerConfig = zodToFactory(IndexingDivinerConfigZod, "toIndexingDivinerConfig");
70
+
71
+ // src/model/Stage.ts
72
+ import { asSchema as asSchema3 } from "@xyo-network/sdk-protocol";
73
+ var DivinerStageSchema = asSchema3("network.xyo.diviner.stage", true);
74
+
75
+ // src/memory/Diviner.ts
76
+ var moduleName = "IndexingDiviner";
77
+ var IndexingDiviner = class extends AbstractDiviner {
78
+ _lastState;
79
+ _pollId;
80
+ get payloadDivinerLimit() {
81
+ return this.config.payloadDivinerLimit ?? 1e3;
82
+ }
83
+ get pollFrequency() {
84
+ return this.config.pollFrequency ?? 1e4;
85
+ }
86
+ /**
87
+ * Works via batched iteration of the source archivist to populate the index.
88
+ * @returns A promise that resolves when the background process is complete
89
+ */
90
+ backgroundDivine = async () => {
91
+ const lastState = await this.retrieveState();
92
+ const indexCandidateDiviner = await this.getIndexingDivinerStage("stateToIndexCandidateDiviner");
93
+ const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine();
94
+ const nextState = results.find(isModuleState);
95
+ const indexCandidates = results.filter((x) => !isModuleState(x));
96
+ const toIndexTransformDiviner = await this.getIndexingDivinerStage("indexCandidateToIndexDiviner");
97
+ const indexes = await toIndexTransformDiviner.divine(indexCandidates);
98
+ const indexArchivist = await this.getArchivistForStore("indexStore");
99
+ await indexArchivist.insert(indexes);
100
+ if (nextState) {
101
+ await this.commitState(nextState);
102
+ }
103
+ };
104
+ /**
105
+ * Commit the internal state of the Diviner process. This is similar
106
+ * to a transaction completion in a database and should only be called
107
+ * when results have been successfully persisted to the appropriate
108
+ * external stores.
109
+ * @param nextState The state to commit
110
+ */
111
+ async commitState(nextState) {
112
+ if (nextState.state.cursor === this._lastState?.state.cursor) return;
113
+ this._lastState = nextState;
114
+ const archivist = await this.getArchivistForStore("stateStore");
115
+ const [bw] = await new BoundWitnessBuilder().payload(nextState).signer(this.account).build();
116
+ await archivist.insert([bw, nextState]);
117
+ }
118
+ async divineHandler(payloads = []) {
119
+ const indexPayloadDiviner = await this.getPayloadDivinerForStore("indexStore");
120
+ const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage("divinerQueryToIndexQueryDiviner");
121
+ const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage("indexQueryResponseToDivinerQueryResponseDiviner");
122
+ const results = (await Promise.all(
123
+ payloads.map(async (payload) => {
124
+ const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload]);
125
+ const indexedResults = await indexPayloadDiviner.divine(indexQuery);
126
+ const response = await Promise.all(
127
+ indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult]))
128
+ );
129
+ return response.flat();
130
+ })
131
+ )).flat();
132
+ return results;
133
+ }
134
+ /**
135
+ * Retrieves the archivist for the specified store
136
+ * @param store The store to retrieve the archivist for
137
+ * @returns The archivist for the specified store
138
+ */
139
+ async getArchivistForStore(store) {
140
+ const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`);
141
+ const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`);
142
+ return ArchivistWrapper.wrap(mod, this.account);
143
+ }
144
+ /**
145
+ * Retrieves the BoundWitness Diviner for the specified store
146
+ * @param store The store to retrieve the BoundWitness Diviner for
147
+ * @returns The BoundWitness Diviner for the specified store
148
+ */
149
+ async getBoundWitnessDivinerForStore(store) {
150
+ const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`);
151
+ const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`);
152
+ return DivinerWrapper.wrap(mod, this.account);
153
+ }
154
+ /**
155
+ * Gets the Diviner for the supplied Indexing Diviner stage
156
+ * @param transform The Indexing Diviner stage
157
+ * @returns The diviner corresponding to the Indexing Diviner stage
158
+ */
159
+ async getIndexingDivinerStage(transform) {
160
+ const nameOrAddress = assertEx(
161
+ this.config?.indexingDivinerStages?.[transform],
162
+ () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`
163
+ );
164
+ const mod = await this.resolve(nameOrAddress);
165
+ return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`);
166
+ }
167
+ /**
168
+ * Retrieves the Payload Diviner for the specified store
169
+ * @param store The store to retrieve the Payload Diviner for
170
+ * @returns The Payload Diviner for the specified store
171
+ */
172
+ async getPayloadDivinerForStore(store) {
173
+ const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`);
174
+ const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`);
175
+ return DivinerWrapper.wrap(mod, this.account);
176
+ }
177
+ /**
178
+ * Retrieves the last state of the Diviner process. Used to recover state after
179
+ * preemptions, reboots, etc.
180
+ */
181
+ async retrieveState() {
182
+ const accountAddress = this.account.address;
183
+ if (this._lastState) return this._lastState;
184
+ let hash = "";
185
+ const diviner = await this.getBoundWitnessDivinerForStore("stateStore");
186
+ const query = new PayloadBuilder({ schema: BoundWitnessDivinerQuerySchema }).fields({
187
+ address: accountAddress,
188
+ limit: 1,
189
+ cursor: SequenceConstants.minLocalSequence,
190
+ order: "desc",
191
+ payload_schemas: [ModuleStateSchema]
192
+ }).build();
193
+ const boundWitnesses = await diviner.divine([query]);
194
+ if (boundWitnesses.length > 0) {
195
+ const boundWitness = boundWitnesses[0];
196
+ if (isBoundWitness(boundWitness)) {
197
+ hash = boundWitness.addresses.map((address, index) => ({ address, index })).filter(({ address }) => address === accountAddress).reduce(
198
+ (prev, curr) => boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev,
199
+ ""
200
+ );
201
+ }
202
+ }
203
+ if (hash !== "") {
204
+ const archivist = await this.getArchivistForStore("stateStore");
205
+ const payload = (await archivist.get([hash])).find(isModuleState);
206
+ if (payload) {
207
+ return payload;
208
+ }
209
+ }
210
+ return void 0;
211
+ }
212
+ async startHandler() {
213
+ await super.startHandler();
214
+ this.poll();
215
+ }
216
+ async stopHandler(_timeout) {
217
+ if (isString(this._pollId)) {
218
+ clearTimeout(this._pollId);
219
+ this._pollId = void 0;
220
+ }
221
+ await super.stopHandler();
222
+ }
223
+ /**
224
+ * Runs the background divine process on a loop with a delay
225
+ * specified by the `config.pollFrequency`
226
+ */
227
+ poll() {
228
+ this._pollId = setTimeoutEx(async () => {
229
+ try {
230
+ await Promise.resolve();
231
+ await this.backgroundDivine();
232
+ } catch (e) {
233
+ console.log(e);
234
+ } finally {
235
+ if (this._pollId) clearTimeoutEx(this._pollId);
236
+ this._pollId = void 0;
237
+ this.poll();
238
+ }
239
+ }, this.pollFrequency);
240
+ }
241
+ };
242
+ __publicField(IndexingDiviner, "allowRandomAccount", true);
243
+ __publicField(IndexingDiviner, "configSchemas", [...__superGet(IndexingDiviner, IndexingDiviner, "configSchemas"), IndexingDivinerConfigSchema]);
244
+ __publicField(IndexingDiviner, "defaultConfigSchema", IndexingDivinerConfigSchema);
245
+ IndexingDiviner = __decorateClass([
246
+ creatableModule()
247
+ ], IndexingDiviner);
248
+ export {
249
+ DivinerStageSchema,
250
+ IndexingDiviner,
251
+ IndexingDivinerConfigSchema,
252
+ IndexingDivinerConfigZod,
253
+ IndexingDivinerSchema,
254
+ asIndexingDivinerConfig,
255
+ isIndexingDivinerConfig,
256
+ toIndexingDivinerConfig
257
+ };
5
258
  //# sourceMappingURL=index.mjs.map
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
- "sources": ["../../src/index.ts"],
4
- "sourcesContent": ["export * from '@xyo-network/diviner-indexing-memory'\nexport * from '@xyo-network/diviner-indexing-model'\nexport * from '@xyo-network/diviner-temporal-indexing'\n"],
5
- "mappings": ";AAAA,cAAc;AACd,cAAc;AACd,cAAc;",
6
- "names": []
3
+ "sources": ["../../src/memory/Diviner.ts", "../../src/model/Config.ts", "../../src/model/Schema.ts", "../../src/model/Stage.ts"],
4
+ "sourcesContent": ["import type { Hash } from '@ariestools/sdk'\nimport {\n assertEx, clearTimeoutEx, isString, setTimeoutEx,\n} from '@ariestools/sdk'\nimport { ArchivistWrapper } from '@xyo-network/sdk/archivist-wrapper'\nimport { AbstractDiviner } from '@xyo-network/sdk/diviner-abstract'\nimport type { BoundWitnessDivinerQueryPayload } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport type {\n DivinerConfig, DivinerInstance, DivinerModule, DivinerModuleEventData, DivinerParams,\n} from '@xyo-network/sdk/diviner-model'\nimport { asDivinerInstance } from '@xyo-network/sdk/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/sdk/diviner-wrapper'\nimport type { AnyConfigSchema, ModuleState } from '@xyo-network/sdk/module-model'\nimport {\n creatableModule, isModuleState, ModuleStateSchema,\n} from '@xyo-network/sdk/module-model'\nimport type {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/sdk-protocol'\nimport {\n BoundWitnessBuilder, isBoundWitness, PayloadBuilder, SequenceConstants,\n} from '@xyo-network/sdk-protocol'\n\nimport type {\n IndexingDivinerConfig,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '../model/index.ts'\nimport { IndexingDivinerConfigSchema } from '../model/index.ts'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = true\n static override readonly configSchemas: Schema[] = [...super.configSchemas, IndexingDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = IndexingDivinerConfigSchema\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit(): number {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency(): number {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter(x => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void> {\n // Don't commit state if no state has changed\n if (nextState.state.cursor === this._lastState?.state.cursor) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).signer(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map(indexedResult => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore): Promise<ArchivistWrapper> {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage): Promise<DivinerInstance> {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n const accountAddress = this.account.address\n if (this._lastState) return this._lastState\n let hash = '' as Hash\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: accountAddress,\n limit: 1,\n cursor: SequenceConstants.minLocalSequence,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === accountAddress)\n\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '' as Hash,\n )\n }\n }\n\n // If we able to located the last state\n if (hash !== '') {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload as WithStorageMeta<ModuleState<IndexingDivinerState>>\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<void> {\n await super.startHandler()\n this.poll()\n }\n\n protected override async stopHandler(_timeout?: number): Promise<void> {\n if (isString(this._pollId)) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n // eslint-disable-next-line @typescript-eslint/no-misused-promises\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n", "import type { EmptyObject, WithAdditional } from '@ariestools/sdk'\nimport {\n zodAsFactory, zodIsFactory, zodToFactory,\n} from '@ariestools/sdk'\nimport type { DivinerConfig, SearchableStorage } from '@xyo-network/sdk/diviner-model'\nimport { DivinerConfigZod } from '@xyo-network/sdk/diviner-model'\nimport {\n asSchema, type Payload, type Schema,\n} from '@xyo-network/sdk-protocol'\nimport * as z from 'zod/mini'\n\nimport { IndexingDivinerSchema } from './Schema.ts'\nimport type { IndexingDivinerStageConfig } from './Stage.ts'\n\nexport const IndexingDivinerConfigSchema: string & {\n readonly __schema: true\n} = asSchema(`${IndexingDivinerSchema}.config`, true)\nexport type IndexingDivinerConfigSchema = typeof IndexingDivinerConfigSchema\n\nexport const IndexingDivinerConfigZod: z.ZodMiniObject<\n Omit<(typeof DivinerConfigZod)['shape'], 'schema'> & {\n indexStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n indexingDivinerStages: z.ZodMiniOptional<z.ZodMiniCustom<IndexingDivinerStageConfig, IndexingDivinerStageConfig>>\n payloadDivinerLimit: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n pollFrequency: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n schema: z.ZodMiniLiteral<IndexingDivinerConfigSchema>\n stateStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n },\n z.core.$strip\n> = z.extend(DivinerConfigZod, {\n indexStore: z.optional(z.custom<SearchableStorage>()),\n indexingDivinerStages: z.optional(z.custom<IndexingDivinerStageConfig>()),\n payloadDivinerLimit: z.optional(z.number()),\n pollFrequency: z.optional(z.number()),\n schema: z.literal(IndexingDivinerConfigSchema),\n stateStore: z.optional(z.custom<SearchableStorage>()),\n})\n\nexport type IndexingDivinerConfigBase = z.infer<typeof IndexingDivinerConfigZod>\n\nexport const isIndexingDivinerConfig: ReturnType<typeof zodIsFactory<IndexingDivinerConfigBase>> = zodIsFactory(IndexingDivinerConfigZod)\nexport const asIndexingDivinerConfig: ReturnType<typeof zodAsFactory<IndexingDivinerConfigBase>>\n = zodAsFactory(IndexingDivinerConfigZod, 'asIndexingDivinerConfig')\nexport const toIndexingDivinerConfig: ReturnType<typeof zodToFactory<IndexingDivinerConfigBase>>\n = zodToFactory(IndexingDivinerConfigZod, 'toIndexingDivinerConfig')\n\nexport type IndexingDivinerConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = DivinerConfig<\n WithAdditional<\n {\n /**\n * Where the diviner should store it's index\n */\n indexStore?: SearchableStorage\n /**\n * Config section for name/address of individual diviner stages\n */\n indexingDivinerStages?: IndexingDivinerStageConfig\n /**\n * The maximum number of payloads to index at a time\n */\n payloadDivinerLimit?: number\n /**\n * How often to poll for new payloads to index\n */\n pollFrequency?: number\n /**\n * The schema for the Diviner config\n */\n schema: IndexingDivinerConfigSchema\n /**\n * Where the diviner should persist its internal state\n */\n stateStore?: SearchableStorage\n },\n TConfig\n >,\n TSchema\n>\n", "import { asSchema } from '@xyo-network/sdk-protocol'\n\nexport const IndexingDivinerSchema: 'network.xyo.diviner.indexing' & {\n readonly __schema: true\n} = asSchema('network.xyo.diviner.indexing', true)\nexport type ImageThumbnailDivinerSchema = typeof IndexingDivinerSchema\n", "export const DivinerStageSchema: 'network.xyo.diviner.stage' & {\n readonly __schema: true\n} = asSchema('network.xyo.diviner.stage', true)\nexport type DivinerStageSchema = typeof DivinerStageSchema\n\nimport type { ModuleIdentifier } from '@xyo-network/sdk/module-model'\nimport { asSchema } from '@xyo-network/sdk-protocol'\n\n/**\n * The diviners for each stage of an indexing diviner\n */\nexport type IndexingDivinerStage\n /**\n * Transforms a diviner query into an index query\n */\n = | 'divinerQueryToIndexQueryDiviner'\n /**\n * Transforms the index candidates into indexes\n */\n | 'indexCandidateToIndexDiviner'\n /**\n * Transforms an index query response into a diviner divine query response\n */\n | 'indexQueryResponseToDivinerQueryResponseDiviner'\n /**\n * Uses the current state to determine the next batch of index candidates\n */\n | 'stateToIndexCandidateDiviner'\n\n/**\n * Config section for declaring each indexing diviner stage\n */\nexport type IndexingDivinerStageConfig = Record<IndexingDivinerStage, ModuleIdentifier>\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;AACA;AAAA,EACE;AAAA,EAAU;AAAA,EAAgB;AAAA,EAAU;AAAA,OAC/B;AACP,SAAS,wBAAwB;AACjC,SAAS,uBAAuB;AAEhC,SAAS,sCAAsC;AAI/C,SAAS,yBAAyB;AAClC,SAAS,sBAAsB;AAE/B;AAAA,EACE;AAAA,EAAiB;AAAA,EAAe;AAAA,OAC3B;AAIP;AAAA,EACE;AAAA,EAAqB;AAAA,EAAgB;AAAA,EAAgB;AAAA,OAChD;;;ACrBP;AAAA,EACE;AAAA,EAAc;AAAA,EAAc;AAAA,OACvB;AAEP,SAAS,wBAAwB;AACjC;AAAA,EACE,YAAAA;AAAA,OACK;AACP,YAAY,OAAO;;;ACTnB,SAAS,gBAAgB;AAElB,IAAM,wBAET,SAAS,gCAAgC,IAAI;;;ADU1C,IAAM,8BAETC,UAAS,GAAG,qBAAqB,WAAW,IAAI;AAG7C,IAAM,2BAUP,SAAO,kBAAkB;AAAA,EAC7B,YAAc,WAAW,SAA0B,CAAC;AAAA,EACpD,uBAAyB,WAAW,SAAmC,CAAC;AAAA,EACxE,qBAAuB,WAAW,SAAO,CAAC;AAAA,EAC1C,eAAiB,WAAW,SAAO,CAAC;AAAA,EACpC,QAAU,UAAQ,2BAA2B;AAAA,EAC7C,YAAc,WAAW,SAA0B,CAAC;AACtD,CAAC;AAIM,IAAM,0BAAsF,aAAa,wBAAwB;AACjI,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;AAC7D,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;;;AEtCpE,SAAS,YAAAC,iBAAgB;AANlB,IAAM,qBAETA,UAAS,6BAA6B,IAAI;;;AHkC9C,IAAM,aAAa;AAGZ,IAAM,kBAAN,cASG,gBAAgD;AAAA,EAKhD;AAAA,EACA;AAAA,EAER,IAAI,sBAA8B;AAChC,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAwB;AAC1B,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,OAAK,CAAC,cAAc,CAAC,CAAC;AAE7D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA6D;AAEvF,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM,OAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,QAAQ,SAAS,EAAE,OAAO,KAAK,OAAO,EAAE,MAAM;AAC3F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,mBAAiB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC7H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAA+C;AAClF,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,eAAe,IAAI,GAAG;AACpH,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAqJ;AAClM,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,yBAAyB,IAAI,GAAG;AAC9H,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAA2D;AACjG,UAAM,gBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAqJ;AAC7L,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,oBAAoB,IAAI,GAAG;AACzH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,UAAM,iBAAiB,KAAK,QAAQ;AACpC,QAAI,KAAK,WAAY,QAAO,KAAK;AACjC,QAAI,OAAO;AACX,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS;AAAA,MACT,OAAO;AAAA,MACP,QAAQ,kBAAkB;AAAA,MAC1B,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,cAAc,EAElD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,oBAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,SAAS,IAAI;AAEf,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAA8B;AACrD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAyB,YAAY,UAAkC;AACrE,QAAI,SAAS,KAAK,OAAO,GAAG;AAC1B,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,UAAM,MAAM,YAAY;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AAEb,SAAK,UAAU,aAAa,YAAY;AACtC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK,QAAS,gBAAe,KAAK,OAAO;AAC7C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;AA3ME,cAVW,iBAUc,sBAAqB;AAC9C,cAXW,iBAWc,iBAA0B,CAAC,GAAG,6CAAM,kBAAe,2BAA2B;AACvG,cAZW,iBAYc,uBAA8B;AAZ5C,kBAAN;AAAA,EADN,gBAAiC;AAAA,GACrB;",
6
+ "names": ["asSchema", "asSchema", "asSchema"]
7
7
  }
@@ -0,0 +1,69 @@
1
+ import { ArchivistWrapper } from '@xyo-network/sdk/archivist-wrapper';
2
+ import { AbstractDiviner } from '@xyo-network/sdk/diviner-abstract';
3
+ import type { DivinerConfig, DivinerInstance, DivinerModule, DivinerModuleEventData, DivinerParams } from '@xyo-network/sdk/diviner-model';
4
+ import { DivinerWrapper } from '@xyo-network/sdk/diviner-wrapper';
5
+ import type { AnyConfigSchema, ModuleState } from '@xyo-network/sdk/module-model';
6
+ import type { Payload, Schema } from '@xyo-network/sdk-protocol';
7
+ import type { IndexingDivinerConfig, IndexingDivinerParams, IndexingDivinerStage, IndexingDivinerState } from '../model/index.ts';
8
+ export type ConfigStoreKey = 'indexStore' | 'stateStore';
9
+ export type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>;
10
+ export declare class IndexingDiviner<TParams extends IndexingDivinerParams = IndexingDivinerParams, TIn extends Payload = Payload, TOut extends Payload = Payload, TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut>> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {
11
+ static readonly allowRandomAccount = true;
12
+ static readonly configSchemas: Schema[];
13
+ static readonly defaultConfigSchema: Schema;
14
+ private _lastState?;
15
+ private _pollId?;
16
+ get payloadDivinerLimit(): number;
17
+ get pollFrequency(): number;
18
+ /**
19
+ * Works via batched iteration of the source archivist to populate the index.
20
+ * @returns A promise that resolves when the background process is complete
21
+ */
22
+ protected backgroundDivine: () => Promise<void>;
23
+ /**
24
+ * Commit the internal state of the Diviner process. This is similar
25
+ * to a transaction completion in a database and should only be called
26
+ * when results have been successfully persisted to the appropriate
27
+ * external stores.
28
+ * @param nextState The state to commit
29
+ */
30
+ protected commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void>;
31
+ protected divineHandler(payloads?: TIn[]): Promise<TOut[]>;
32
+ /**
33
+ * Retrieves the archivist for the specified store
34
+ * @param store The store to retrieve the archivist for
35
+ * @returns The archivist for the specified store
36
+ */
37
+ protected getArchivistForStore(store: ConfigStore): Promise<ArchivistWrapper>;
38
+ /**
39
+ * Retrieves the BoundWitness Diviner for the specified store
40
+ * @param store The store to retrieve the BoundWitness Diviner for
41
+ * @returns The BoundWitness Diviner for the specified store
42
+ */
43
+ protected getBoundWitnessDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>>;
44
+ /**
45
+ * Gets the Diviner for the supplied Indexing Diviner stage
46
+ * @param transform The Indexing Diviner stage
47
+ * @returns The diviner corresponding to the Indexing Diviner stage
48
+ */
49
+ protected getIndexingDivinerStage(transform: IndexingDivinerStage): Promise<DivinerInstance>;
50
+ /**
51
+ * Retrieves the Payload Diviner for the specified store
52
+ * @param store The store to retrieve the Payload Diviner for
53
+ * @returns The Payload Diviner for the specified store
54
+ */
55
+ protected getPayloadDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>>;
56
+ /**
57
+ * Retrieves the last state of the Diviner process. Used to recover state after
58
+ * preemptions, reboots, etc.
59
+ */
60
+ protected retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined>;
61
+ protected startHandler(): Promise<void>;
62
+ protected stopHandler(_timeout?: number): Promise<void>;
63
+ /**
64
+ * Runs the background divine process on a loop with a delay
65
+ * specified by the `config.pollFrequency`
66
+ */
67
+ private poll;
68
+ }
69
+ //# sourceMappingURL=Diviner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/memory/Diviner.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAA;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAA;AAGnE,OAAO,KAAK,EACV,aAAa,EAAE,eAAe,EAAE,aAAa,EAAE,sBAAsB,EAAE,aAAa,EACrF,MAAM,gCAAgC,CAAA;AAEvC,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAA;AACjE,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAA;AAIjF,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAChB,MAAM,2BAA2B,CAAA;AAKlC,OAAO,KAAK,EACV,qBAAqB,EACrB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,mBAAmB,CAAA;AAG1B,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,QAAO;IAClD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAwD;IACxG,gBAAyB,mBAAmB,EAAE,MAAM,CAA8B;IAElF,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED,IAAI,aAAa,IAAI,MAAM,CAE1B;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,OAAO,CAAC,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;cAS/D,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAMnF;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,aAAa,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,EAAE,sBAAsB,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAMnM;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB,GAAG,OAAO,CAAC,eAAe,CAAC;IASlG;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,aAAa,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,EAAE,sBAAsB,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAM9L;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cA0C9D,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;cAK7B,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQtE;;;OAGG;IACH,OAAO,CAAC,IAAI;CAeb"}
@@ -0,0 +1,2 @@
1
+ export * from './Diviner.ts';
2
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/memory/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAA"}
@@ -0,0 +1,2 @@
1
+ export * from './memory/index.ts';
2
+ //# sourceMappingURL=memory.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../src/memory.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAA"}
@@ -0,0 +1,247 @@
1
+ var __defProp = Object.defineProperty;
2
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
3
+ var __getProtoOf = Object.getPrototypeOf;
4
+ var __reflectGet = Reflect.get;
5
+ var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
6
+ var __decorateClass = (decorators, target, key, kind) => {
7
+ var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
8
+ for (var i = decorators.length - 1, decorator; i >= 0; i--)
9
+ if (decorator = decorators[i])
10
+ result = (kind ? decorator(target, key, result) : decorator(result)) || result;
11
+ if (kind && result) __defProp(target, key, result);
12
+ return result;
13
+ };
14
+ var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
15
+ var __superGet = (cls, obj, key) => __reflectGet(__getProtoOf(cls), key, obj);
16
+
17
+ // src/memory/Diviner.ts
18
+ import {
19
+ assertEx,
20
+ clearTimeoutEx,
21
+ isString,
22
+ setTimeoutEx
23
+ } from "@ariestools/sdk";
24
+ import { ArchivistWrapper } from "@xyo-network/sdk/archivist-wrapper";
25
+ import { AbstractDiviner } from "@xyo-network/sdk/diviner-abstract";
26
+ import { BoundWitnessDivinerQuerySchema } from "@xyo-network/sdk/diviner-boundwitness/model";
27
+ import { asDivinerInstance } from "@xyo-network/sdk/diviner-model";
28
+ import { DivinerWrapper } from "@xyo-network/sdk/diviner-wrapper";
29
+ import {
30
+ creatableModule,
31
+ isModuleState,
32
+ ModuleStateSchema
33
+ } from "@xyo-network/sdk/module-model";
34
+ import {
35
+ BoundWitnessBuilder,
36
+ isBoundWitness,
37
+ PayloadBuilder,
38
+ SequenceConstants
39
+ } from "@xyo-network/sdk-protocol";
40
+
41
+ // src/model/Config.ts
42
+ import {
43
+ zodAsFactory,
44
+ zodIsFactory,
45
+ zodToFactory
46
+ } from "@ariestools/sdk";
47
+ import { DivinerConfigZod } from "@xyo-network/sdk/diviner-model";
48
+ import {
49
+ asSchema as asSchema2
50
+ } from "@xyo-network/sdk-protocol";
51
+ import * as z from "zod/mini";
52
+
53
+ // src/model/Schema.ts
54
+ import { asSchema } from "@xyo-network/sdk-protocol";
55
+ var IndexingDivinerSchema = asSchema("network.xyo.diviner.indexing", true);
56
+
57
+ // src/model/Config.ts
58
+ var IndexingDivinerConfigSchema = asSchema2(`${IndexingDivinerSchema}.config`, true);
59
+ var IndexingDivinerConfigZod = z.extend(DivinerConfigZod, {
60
+ indexStore: z.optional(z.custom()),
61
+ indexingDivinerStages: z.optional(z.custom()),
62
+ payloadDivinerLimit: z.optional(z.number()),
63
+ pollFrequency: z.optional(z.number()),
64
+ schema: z.literal(IndexingDivinerConfigSchema),
65
+ stateStore: z.optional(z.custom())
66
+ });
67
+ var isIndexingDivinerConfig = zodIsFactory(IndexingDivinerConfigZod);
68
+ var asIndexingDivinerConfig = zodAsFactory(IndexingDivinerConfigZod, "asIndexingDivinerConfig");
69
+ var toIndexingDivinerConfig = zodToFactory(IndexingDivinerConfigZod, "toIndexingDivinerConfig");
70
+
71
+ // src/memory/Diviner.ts
72
+ var moduleName = "IndexingDiviner";
73
+ var IndexingDiviner = class extends AbstractDiviner {
74
+ _lastState;
75
+ _pollId;
76
+ get payloadDivinerLimit() {
77
+ return this.config.payloadDivinerLimit ?? 1e3;
78
+ }
79
+ get pollFrequency() {
80
+ return this.config.pollFrequency ?? 1e4;
81
+ }
82
+ /**
83
+ * Works via batched iteration of the source archivist to populate the index.
84
+ * @returns A promise that resolves when the background process is complete
85
+ */
86
+ backgroundDivine = async () => {
87
+ const lastState = await this.retrieveState();
88
+ const indexCandidateDiviner = await this.getIndexingDivinerStage("stateToIndexCandidateDiviner");
89
+ const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine();
90
+ const nextState = results.find(isModuleState);
91
+ const indexCandidates = results.filter((x) => !isModuleState(x));
92
+ const toIndexTransformDiviner = await this.getIndexingDivinerStage("indexCandidateToIndexDiviner");
93
+ const indexes = await toIndexTransformDiviner.divine(indexCandidates);
94
+ const indexArchivist = await this.getArchivistForStore("indexStore");
95
+ await indexArchivist.insert(indexes);
96
+ if (nextState) {
97
+ await this.commitState(nextState);
98
+ }
99
+ };
100
+ /**
101
+ * Commit the internal state of the Diviner process. This is similar
102
+ * to a transaction completion in a database and should only be called
103
+ * when results have been successfully persisted to the appropriate
104
+ * external stores.
105
+ * @param nextState The state to commit
106
+ */
107
+ async commitState(nextState) {
108
+ if (nextState.state.cursor === this._lastState?.state.cursor) return;
109
+ this._lastState = nextState;
110
+ const archivist = await this.getArchivistForStore("stateStore");
111
+ const [bw] = await new BoundWitnessBuilder().payload(nextState).signer(this.account).build();
112
+ await archivist.insert([bw, nextState]);
113
+ }
114
+ async divineHandler(payloads = []) {
115
+ const indexPayloadDiviner = await this.getPayloadDivinerForStore("indexStore");
116
+ const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage("divinerQueryToIndexQueryDiviner");
117
+ const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage("indexQueryResponseToDivinerQueryResponseDiviner");
118
+ const results = (await Promise.all(
119
+ payloads.map(async (payload) => {
120
+ const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload]);
121
+ const indexedResults = await indexPayloadDiviner.divine(indexQuery);
122
+ const response = await Promise.all(
123
+ indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult]))
124
+ );
125
+ return response.flat();
126
+ })
127
+ )).flat();
128
+ return results;
129
+ }
130
+ /**
131
+ * Retrieves the archivist for the specified store
132
+ * @param store The store to retrieve the archivist for
133
+ * @returns The archivist for the specified store
134
+ */
135
+ async getArchivistForStore(store) {
136
+ const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`);
137
+ const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`);
138
+ return ArchivistWrapper.wrap(mod, this.account);
139
+ }
140
+ /**
141
+ * Retrieves the BoundWitness Diviner for the specified store
142
+ * @param store The store to retrieve the BoundWitness Diviner for
143
+ * @returns The BoundWitness Diviner for the specified store
144
+ */
145
+ async getBoundWitnessDivinerForStore(store) {
146
+ const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`);
147
+ const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`);
148
+ return DivinerWrapper.wrap(mod, this.account);
149
+ }
150
+ /**
151
+ * Gets the Diviner for the supplied Indexing Diviner stage
152
+ * @param transform The Indexing Diviner stage
153
+ * @returns The diviner corresponding to the Indexing Diviner stage
154
+ */
155
+ async getIndexingDivinerStage(transform) {
156
+ const nameOrAddress = assertEx(
157
+ this.config?.indexingDivinerStages?.[transform],
158
+ () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`
159
+ );
160
+ const mod = await this.resolve(nameOrAddress);
161
+ return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`);
162
+ }
163
+ /**
164
+ * Retrieves the Payload Diviner for the specified store
165
+ * @param store The store to retrieve the Payload Diviner for
166
+ * @returns The Payload Diviner for the specified store
167
+ */
168
+ async getPayloadDivinerForStore(store) {
169
+ const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`);
170
+ const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`);
171
+ return DivinerWrapper.wrap(mod, this.account);
172
+ }
173
+ /**
174
+ * Retrieves the last state of the Diviner process. Used to recover state after
175
+ * preemptions, reboots, etc.
176
+ */
177
+ async retrieveState() {
178
+ const accountAddress = this.account.address;
179
+ if (this._lastState) return this._lastState;
180
+ let hash = "";
181
+ const diviner = await this.getBoundWitnessDivinerForStore("stateStore");
182
+ const query = new PayloadBuilder({ schema: BoundWitnessDivinerQuerySchema }).fields({
183
+ address: accountAddress,
184
+ limit: 1,
185
+ cursor: SequenceConstants.minLocalSequence,
186
+ order: "desc",
187
+ payload_schemas: [ModuleStateSchema]
188
+ }).build();
189
+ const boundWitnesses = await diviner.divine([query]);
190
+ if (boundWitnesses.length > 0) {
191
+ const boundWitness = boundWitnesses[0];
192
+ if (isBoundWitness(boundWitness)) {
193
+ hash = boundWitness.addresses.map((address, index) => ({ address, index })).filter(({ address }) => address === accountAddress).reduce(
194
+ (prev, curr) => boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev,
195
+ ""
196
+ );
197
+ }
198
+ }
199
+ if (hash !== "") {
200
+ const archivist = await this.getArchivistForStore("stateStore");
201
+ const payload = (await archivist.get([hash])).find(isModuleState);
202
+ if (payload) {
203
+ return payload;
204
+ }
205
+ }
206
+ return void 0;
207
+ }
208
+ async startHandler() {
209
+ await super.startHandler();
210
+ this.poll();
211
+ }
212
+ async stopHandler(_timeout) {
213
+ if (isString(this._pollId)) {
214
+ clearTimeout(this._pollId);
215
+ this._pollId = void 0;
216
+ }
217
+ await super.stopHandler();
218
+ }
219
+ /**
220
+ * Runs the background divine process on a loop with a delay
221
+ * specified by the `config.pollFrequency`
222
+ */
223
+ poll() {
224
+ this._pollId = setTimeoutEx(async () => {
225
+ try {
226
+ await Promise.resolve();
227
+ await this.backgroundDivine();
228
+ } catch (e) {
229
+ console.log(e);
230
+ } finally {
231
+ if (this._pollId) clearTimeoutEx(this._pollId);
232
+ this._pollId = void 0;
233
+ this.poll();
234
+ }
235
+ }, this.pollFrequency);
236
+ }
237
+ };
238
+ __publicField(IndexingDiviner, "allowRandomAccount", true);
239
+ __publicField(IndexingDiviner, "configSchemas", [...__superGet(IndexingDiviner, IndexingDiviner, "configSchemas"), IndexingDivinerConfigSchema]);
240
+ __publicField(IndexingDiviner, "defaultConfigSchema", IndexingDivinerConfigSchema);
241
+ IndexingDiviner = __decorateClass([
242
+ creatableModule()
243
+ ], IndexingDiviner);
244
+ export {
245
+ IndexingDiviner
246
+ };
247
+ //# sourceMappingURL=memory.mjs.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/memory/Diviner.ts", "../../src/model/Config.ts", "../../src/model/Schema.ts"],
4
+ "sourcesContent": ["import type { Hash } from '@ariestools/sdk'\nimport {\n assertEx, clearTimeoutEx, isString, setTimeoutEx,\n} from '@ariestools/sdk'\nimport { ArchivistWrapper } from '@xyo-network/sdk/archivist-wrapper'\nimport { AbstractDiviner } from '@xyo-network/sdk/diviner-abstract'\nimport type { BoundWitnessDivinerQueryPayload } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport type {\n DivinerConfig, DivinerInstance, DivinerModule, DivinerModuleEventData, DivinerParams,\n} from '@xyo-network/sdk/diviner-model'\nimport { asDivinerInstance } from '@xyo-network/sdk/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/sdk/diviner-wrapper'\nimport type { AnyConfigSchema, ModuleState } from '@xyo-network/sdk/module-model'\nimport {\n creatableModule, isModuleState, ModuleStateSchema,\n} from '@xyo-network/sdk/module-model'\nimport type {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/sdk-protocol'\nimport {\n BoundWitnessBuilder, isBoundWitness, PayloadBuilder, SequenceConstants,\n} from '@xyo-network/sdk-protocol'\n\nimport type {\n IndexingDivinerConfig,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '../model/index.ts'\nimport { IndexingDivinerConfigSchema } from '../model/index.ts'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = true\n static override readonly configSchemas: Schema[] = [...super.configSchemas, IndexingDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = IndexingDivinerConfigSchema\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit(): number {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency(): number {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter(x => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void> {\n // Don't commit state if no state has changed\n if (nextState.state.cursor === this._lastState?.state.cursor) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).signer(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map(indexedResult => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore): Promise<ArchivistWrapper> {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage): Promise<DivinerInstance> {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n const accountAddress = this.account.address\n if (this._lastState) return this._lastState\n let hash = '' as Hash\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: accountAddress,\n limit: 1,\n cursor: SequenceConstants.minLocalSequence,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === accountAddress)\n\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '' as Hash,\n )\n }\n }\n\n // If we able to located the last state\n if (hash !== '') {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload as WithStorageMeta<ModuleState<IndexingDivinerState>>\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<void> {\n await super.startHandler()\n this.poll()\n }\n\n protected override async stopHandler(_timeout?: number): Promise<void> {\n if (isString(this._pollId)) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n // eslint-disable-next-line @typescript-eslint/no-misused-promises\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n", "import type { EmptyObject, WithAdditional } from '@ariestools/sdk'\nimport {\n zodAsFactory, zodIsFactory, zodToFactory,\n} from '@ariestools/sdk'\nimport type { DivinerConfig, SearchableStorage } from '@xyo-network/sdk/diviner-model'\nimport { DivinerConfigZod } from '@xyo-network/sdk/diviner-model'\nimport {\n asSchema, type Payload, type Schema,\n} from '@xyo-network/sdk-protocol'\nimport * as z from 'zod/mini'\n\nimport { IndexingDivinerSchema } from './Schema.ts'\nimport type { IndexingDivinerStageConfig } from './Stage.ts'\n\nexport const IndexingDivinerConfigSchema: string & {\n readonly __schema: true\n} = asSchema(`${IndexingDivinerSchema}.config`, true)\nexport type IndexingDivinerConfigSchema = typeof IndexingDivinerConfigSchema\n\nexport const IndexingDivinerConfigZod: z.ZodMiniObject<\n Omit<(typeof DivinerConfigZod)['shape'], 'schema'> & {\n indexStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n indexingDivinerStages: z.ZodMiniOptional<z.ZodMiniCustom<IndexingDivinerStageConfig, IndexingDivinerStageConfig>>\n payloadDivinerLimit: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n pollFrequency: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n schema: z.ZodMiniLiteral<IndexingDivinerConfigSchema>\n stateStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n },\n z.core.$strip\n> = z.extend(DivinerConfigZod, {\n indexStore: z.optional(z.custom<SearchableStorage>()),\n indexingDivinerStages: z.optional(z.custom<IndexingDivinerStageConfig>()),\n payloadDivinerLimit: z.optional(z.number()),\n pollFrequency: z.optional(z.number()),\n schema: z.literal(IndexingDivinerConfigSchema),\n stateStore: z.optional(z.custom<SearchableStorage>()),\n})\n\nexport type IndexingDivinerConfigBase = z.infer<typeof IndexingDivinerConfigZod>\n\nexport const isIndexingDivinerConfig: ReturnType<typeof zodIsFactory<IndexingDivinerConfigBase>> = zodIsFactory(IndexingDivinerConfigZod)\nexport const asIndexingDivinerConfig: ReturnType<typeof zodAsFactory<IndexingDivinerConfigBase>>\n = zodAsFactory(IndexingDivinerConfigZod, 'asIndexingDivinerConfig')\nexport const toIndexingDivinerConfig: ReturnType<typeof zodToFactory<IndexingDivinerConfigBase>>\n = zodToFactory(IndexingDivinerConfigZod, 'toIndexingDivinerConfig')\n\nexport type IndexingDivinerConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = DivinerConfig<\n WithAdditional<\n {\n /**\n * Where the diviner should store it's index\n */\n indexStore?: SearchableStorage\n /**\n * Config section for name/address of individual diviner stages\n */\n indexingDivinerStages?: IndexingDivinerStageConfig\n /**\n * The maximum number of payloads to index at a time\n */\n payloadDivinerLimit?: number\n /**\n * How often to poll for new payloads to index\n */\n pollFrequency?: number\n /**\n * The schema for the Diviner config\n */\n schema: IndexingDivinerConfigSchema\n /**\n * Where the diviner should persist its internal state\n */\n stateStore?: SearchableStorage\n },\n TConfig\n >,\n TSchema\n>\n", "import { asSchema } from '@xyo-network/sdk-protocol'\n\nexport const IndexingDivinerSchema: 'network.xyo.diviner.indexing' & {\n readonly __schema: true\n} = asSchema('network.xyo.diviner.indexing', true)\nexport type ImageThumbnailDivinerSchema = typeof IndexingDivinerSchema\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;AACA;AAAA,EACE;AAAA,EAAU;AAAA,EAAgB;AAAA,EAAU;AAAA,OAC/B;AACP,SAAS,wBAAwB;AACjC,SAAS,uBAAuB;AAEhC,SAAS,sCAAsC;AAI/C,SAAS,yBAAyB;AAClC,SAAS,sBAAsB;AAE/B;AAAA,EACE;AAAA,EAAiB;AAAA,EAAe;AAAA,OAC3B;AAIP;AAAA,EACE;AAAA,EAAqB;AAAA,EAAgB;AAAA,EAAgB;AAAA,OAChD;;;ACrBP;AAAA,EACE;AAAA,EAAc;AAAA,EAAc;AAAA,OACvB;AAEP,SAAS,wBAAwB;AACjC;AAAA,EACE,YAAAA;AAAA,OACK;AACP,YAAY,OAAO;;;ACTnB,SAAS,gBAAgB;AAElB,IAAM,wBAET,SAAS,gCAAgC,IAAI;;;ADU1C,IAAM,8BAETC,UAAS,GAAG,qBAAqB,WAAW,IAAI;AAG7C,IAAM,2BAUP,SAAO,kBAAkB;AAAA,EAC7B,YAAc,WAAW,SAA0B,CAAC;AAAA,EACpD,uBAAyB,WAAW,SAAmC,CAAC;AAAA,EACxE,qBAAuB,WAAW,SAAO,CAAC;AAAA,EAC1C,eAAiB,WAAW,SAAO,CAAC;AAAA,EACpC,QAAU,UAAQ,2BAA2B;AAAA,EAC7C,YAAc,WAAW,SAA0B,CAAC;AACtD,CAAC;AAIM,IAAM,0BAAsF,aAAa,wBAAwB;AACjI,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;AAC7D,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;;;ADRpE,IAAM,aAAa;AAGZ,IAAM,kBAAN,cASG,gBAAgD;AAAA,EAKhD;AAAA,EACA;AAAA,EAER,IAAI,sBAA8B;AAChC,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAwB;AAC1B,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,OAAK,CAAC,cAAc,CAAC,CAAC;AAE7D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA6D;AAEvF,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM,OAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,QAAQ,SAAS,EAAE,OAAO,KAAK,OAAO,EAAE,MAAM;AAC3F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,mBAAiB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC7H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAA+C;AAClF,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,eAAe,IAAI,GAAG;AACpH,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAqJ;AAClM,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,yBAAyB,IAAI,GAAG;AAC9H,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAA2D;AACjG,UAAM,gBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAqJ;AAC7L,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,oBAAoB,IAAI,GAAG;AACzH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,UAAM,iBAAiB,KAAK,QAAQ;AACpC,QAAI,KAAK,WAAY,QAAO,KAAK;AACjC,QAAI,OAAO;AACX,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS;AAAA,MACT,OAAO;AAAA,MACP,QAAQ,kBAAkB;AAAA,MAC1B,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,cAAc,EAElD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,oBAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,SAAS,IAAI;AAEf,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAA8B;AACrD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAyB,YAAY,UAAkC;AACrE,QAAI,SAAS,KAAK,OAAO,GAAG;AAC1B,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,UAAM,MAAM,YAAY;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AAEb,SAAK,UAAU,aAAa,YAAY;AACtC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK,QAAS,gBAAe,KAAK,OAAO;AAC7C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;AA3ME,cAVW,iBAUc,sBAAqB;AAC9C,cAXW,iBAWc,iBAA0B,CAAC,GAAG,6CAAM,kBAAe,2BAA2B;AACvG,cAZW,iBAYc,uBAA8B;AAZ5C,kBAAN;AAAA,EADN,gBAAiC;AAAA,GACrB;",
6
+ "names": ["asSchema", "asSchema"]
7
+ }
@@ -0,0 +1,50 @@
1
+ import type { EmptyObject, WithAdditional } from '@ariestools/sdk';
2
+ import { zodAsFactory, zodIsFactory, zodToFactory } from '@ariestools/sdk';
3
+ import type { DivinerConfig, SearchableStorage } from '@xyo-network/sdk/diviner-model';
4
+ import { DivinerConfigZod } from '@xyo-network/sdk/diviner-model';
5
+ import { type Payload, type Schema } from '@xyo-network/sdk-protocol';
6
+ import * as z from 'zod/mini';
7
+ import type { IndexingDivinerStageConfig } from './Stage.ts';
8
+ export declare const IndexingDivinerConfigSchema: string & {
9
+ readonly __schema: true;
10
+ };
11
+ export type IndexingDivinerConfigSchema = typeof IndexingDivinerConfigSchema;
12
+ export declare const IndexingDivinerConfigZod: z.ZodMiniObject<Omit<(typeof DivinerConfigZod)['shape'], 'schema'> & {
13
+ indexStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>;
14
+ indexingDivinerStages: z.ZodMiniOptional<z.ZodMiniCustom<IndexingDivinerStageConfig, IndexingDivinerStageConfig>>;
15
+ payloadDivinerLimit: z.ZodMiniOptional<z.ZodMiniNumber<number>>;
16
+ pollFrequency: z.ZodMiniOptional<z.ZodMiniNumber<number>>;
17
+ schema: z.ZodMiniLiteral<IndexingDivinerConfigSchema>;
18
+ stateStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>;
19
+ }, z.core.$strip>;
20
+ export type IndexingDivinerConfigBase = z.infer<typeof IndexingDivinerConfigZod>;
21
+ export declare const isIndexingDivinerConfig: ReturnType<typeof zodIsFactory<IndexingDivinerConfigBase>>;
22
+ export declare const asIndexingDivinerConfig: ReturnType<typeof zodAsFactory<IndexingDivinerConfigBase>>;
23
+ export declare const toIndexingDivinerConfig: ReturnType<typeof zodToFactory<IndexingDivinerConfigBase>>;
24
+ export type IndexingDivinerConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = DivinerConfig<WithAdditional<{
25
+ /**
26
+ * Where the diviner should store it's index
27
+ */
28
+ indexStore?: SearchableStorage;
29
+ /**
30
+ * Config section for name/address of individual diviner stages
31
+ */
32
+ indexingDivinerStages?: IndexingDivinerStageConfig;
33
+ /**
34
+ * The maximum number of payloads to index at a time
35
+ */
36
+ payloadDivinerLimit?: number;
37
+ /**
38
+ * How often to poll for new payloads to index
39
+ */
40
+ pollFrequency?: number;
41
+ /**
42
+ * The schema for the Diviner config
43
+ */
44
+ schema: IndexingDivinerConfigSchema;
45
+ /**
46
+ * Where the diviner should persist its internal state
47
+ */
48
+ stateStore?: SearchableStorage;
49
+ }, TConfig>, TSchema>;
50
+ //# sourceMappingURL=Config.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Config.d.ts","sourceRoot":"","sources":["../../../src/model/Config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAClE,OAAO,EACL,YAAY,EAAE,YAAY,EAAE,YAAY,EACzC,MAAM,iBAAiB,CAAA;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAA;AACtF,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AACjE,OAAO,EACK,KAAK,OAAO,EAAE,KAAK,MAAM,EACpC,MAAM,2BAA2B,CAAA;AAClC,OAAO,KAAK,CAAC,MAAM,UAAU,CAAA;AAG7B,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAA;AAE5D,eAAO,MAAM,2BAA2B,EAAE,MAAM,GAAG;IACjD,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAA;CAC4B,CAAA;AACrD,MAAM,MAAM,2BAA2B,GAAG,OAAO,2BAA2B,CAAA;AAE5E,eAAO,MAAM,wBAAwB,EAAE,CAAC,CAAC,aAAa,CACpD,IAAI,CAAC,CAAC,OAAO,gBAAgB,CAAC,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,GAAG;IACnD,UAAU,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC,CAAA;IACpF,qBAAqB,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,CAAC,0BAA0B,EAAE,0BAA0B,CAAC,CAAC,CAAA;IACjH,mBAAmB,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAA;IAC/D,aAAa,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,CAAC,cAAc,CAAC,2BAA2B,CAAC,CAAA;IACrD,UAAU,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC,CAAA;CACrF,EACD,CAAC,CAAC,IAAI,CAAC,MAAM,CAQb,CAAA;AAEF,MAAM,MAAM,yBAAyB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAA;AAEhF,eAAO,MAAM,uBAAuB,EAAE,UAAU,CAAC,OAAO,YAAY,CAAC,yBAAyB,CAAC,CAA0C,CAAA;AACzI,eAAO,MAAM,uBAAuB,EAAE,UAAU,CAAC,OAAO,YAAY,CAAC,yBAAyB,CAAC,CAC1B,CAAA;AACrE,eAAO,MAAM,uBAAuB,EAAE,UAAU,CAAC,OAAO,YAAY,CAAC,yBAAyB,CAAC,CAC1B,CAAA;AAErE,MAAM,MAAM,qBAAqB,CAAC,OAAO,SAAS,OAAO,GAAG,WAAW,GAAG,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,MAAM,GAAG,IAAI,GAAG,IAAI,IAAI,aAAa,CAC1I,cAAc,CACZ;IACE;;OAEG;IACH,UAAU,CAAC,EAAE,iBAAiB,CAAA;IAC9B;;OAEG;IACH,qBAAqB,CAAC,EAAE,0BAA0B,CAAA;IAClD;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,MAAM,EAAE,2BAA2B,CAAA;IACnC;;OAEG;IACH,UAAU,CAAC,EAAE,iBAAiB,CAAA;CAC/B,EACD,OAAO,CACR,EACD,OAAO,CACR,CAAA"}
@@ -0,0 +1,7 @@
1
+ import type { Labels } from '@xyo-network/sdk/module-model';
2
+ import type { DivinerStageSchema, IndexingDivinerStage } from './Stage.ts';
3
+ /**
4
+ * Labels for Indexing Diviner Stage Diviners
5
+ */
6
+ export type IndexingDivinerStageLabels = Labels & Record<DivinerStageSchema, IndexingDivinerStage>;
7
+ //# sourceMappingURL=Labels.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Labels.d.ts","sourceRoot":"","sources":["../../../src/model/Labels.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,+BAA+B,CAAA;AAE3D,OAAO,KAAK,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAE1E;;GAEG;AACH,MAAM,MAAM,0BAA0B,GAAG,MAAM,GAAG,MAAM,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAA"}
@@ -0,0 +1,5 @@
1
+ import type { DivinerParams } from '@xyo-network/sdk/diviner-model';
2
+ import type { AnyConfigSchema } from '@xyo-network/sdk/module-model';
3
+ import type { IndexingDivinerConfig } from './Config.ts';
4
+ export type IndexingDivinerParams = DivinerParams<AnyConfigSchema<IndexingDivinerConfig>>;
5
+ //# sourceMappingURL=Params.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Params.d.ts","sourceRoot":"","sources":["../../../src/model/Params.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAA;AACnE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAEpE,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAA;AAExD,MAAM,MAAM,qBAAqB,GAAG,aAAa,CAAC,eAAe,CAAC,qBAAqB,CAAC,CAAC,CAAA"}
@@ -0,0 +1,5 @@
1
+ export declare const IndexingDivinerSchema: 'network.xyo.diviner.indexing' & {
2
+ readonly __schema: true;
3
+ };
4
+ export type ImageThumbnailDivinerSchema = typeof IndexingDivinerSchema;
5
+ //# sourceMappingURL=Schema.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Schema.d.ts","sourceRoot":"","sources":["../../../src/model/Schema.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,qBAAqB,EAAE,8BAA8B,GAAG;IACnE,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAA;CACyB,CAAA;AAClD,MAAM,MAAM,2BAA2B,GAAG,OAAO,qBAAqB,CAAA"}
@@ -0,0 +1,26 @@
1
+ export declare const DivinerStageSchema: 'network.xyo.diviner.stage' & {
2
+ readonly __schema: true;
3
+ };
4
+ export type DivinerStageSchema = typeof DivinerStageSchema;
5
+ import type { ModuleIdentifier } from '@xyo-network/sdk/module-model';
6
+ /**
7
+ * The diviners for each stage of an indexing diviner
8
+ */
9
+ export type IndexingDivinerStage = 'divinerQueryToIndexQueryDiviner'
10
+ /**
11
+ * Transforms the index candidates into indexes
12
+ */
13
+ | 'indexCandidateToIndexDiviner'
14
+ /**
15
+ * Transforms an index query response into a diviner divine query response
16
+ */
17
+ | 'indexQueryResponseToDivinerQueryResponseDiviner'
18
+ /**
19
+ * Uses the current state to determine the next batch of index candidates
20
+ */
21
+ | 'stateToIndexCandidateDiviner';
22
+ /**
23
+ * Config section for declaring each indexing diviner stage
24
+ */
25
+ export type IndexingDivinerStageConfig = Record<IndexingDivinerStage, ModuleIdentifier>;
26
+ //# sourceMappingURL=Stage.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Stage.d.ts","sourceRoot":"","sources":["../../../src/model/Stage.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,kBAAkB,EAAE,2BAA2B,GAAG;IAC7D,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAA;CACsB,CAAA;AAC/C,MAAM,MAAM,kBAAkB,GAAG,OAAO,kBAAkB,CAAA;AAE1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAA;AAGrE;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAI1B,iCAAiC;AACrC;;GAEG;GACC,8BAA8B;AAClC;;GAEG;GACC,iDAAiD;AACrD;;GAEG;GACC,8BAA8B,CAAA;AAEpC;;GAEG;AACH,MAAM,MAAM,0BAA0B,GAAG,MAAM,CAAC,oBAAoB,EAAE,gBAAgB,CAAC,CAAA"}
@@ -0,0 +1,6 @@
1
+ import type { StateDictionary } from '@xyo-network/sdk/module-model';
2
+ import type { Sequence } from '@xyo-network/sdk-protocol';
3
+ export type IndexingDivinerState = StateDictionary & {
4
+ cursor: Sequence;
5
+ };
6
+ //# sourceMappingURL=State.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"State.d.ts","sourceRoot":"","sources":["../../../src/model/State.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AACpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AAEzD,MAAM,MAAM,oBAAoB,GAAG,eAAe,GAAG;IACnD,MAAM,EAAE,QAAQ,CAAA;CACjB,CAAA"}
@@ -0,0 +1,7 @@
1
+ export * from './Config.ts';
2
+ export * from './Labels.ts';
3
+ export * from './Params.ts';
4
+ export * from './Schema.ts';
5
+ export * from './Stage.ts';
6
+ export * from './State.ts';
7
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/model/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAC3B,cAAc,YAAY,CAAA;AAC1B,cAAc,YAAY,CAAA"}
@@ -0,0 +1,2 @@
1
+ export * from './model/index.ts';
2
+ //# sourceMappingURL=model.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../../src/model.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAA"}
@@ -0,0 +1,43 @@
1
+ // src/model/Config.ts
2
+ import {
3
+ zodAsFactory,
4
+ zodIsFactory,
5
+ zodToFactory
6
+ } from "@ariestools/sdk";
7
+ import { DivinerConfigZod } from "@xyo-network/sdk/diviner-model";
8
+ import {
9
+ asSchema as asSchema2
10
+ } from "@xyo-network/sdk-protocol";
11
+ import * as z from "zod/mini";
12
+
13
+ // src/model/Schema.ts
14
+ import { asSchema } from "@xyo-network/sdk-protocol";
15
+ var IndexingDivinerSchema = asSchema("network.xyo.diviner.indexing", true);
16
+
17
+ // src/model/Config.ts
18
+ var IndexingDivinerConfigSchema = asSchema2(`${IndexingDivinerSchema}.config`, true);
19
+ var IndexingDivinerConfigZod = z.extend(DivinerConfigZod, {
20
+ indexStore: z.optional(z.custom()),
21
+ indexingDivinerStages: z.optional(z.custom()),
22
+ payloadDivinerLimit: z.optional(z.number()),
23
+ pollFrequency: z.optional(z.number()),
24
+ schema: z.literal(IndexingDivinerConfigSchema),
25
+ stateStore: z.optional(z.custom())
26
+ });
27
+ var isIndexingDivinerConfig = zodIsFactory(IndexingDivinerConfigZod);
28
+ var asIndexingDivinerConfig = zodAsFactory(IndexingDivinerConfigZod, "asIndexingDivinerConfig");
29
+ var toIndexingDivinerConfig = zodToFactory(IndexingDivinerConfigZod, "toIndexingDivinerConfig");
30
+
31
+ // src/model/Stage.ts
32
+ import { asSchema as asSchema3 } from "@xyo-network/sdk-protocol";
33
+ var DivinerStageSchema = asSchema3("network.xyo.diviner.stage", true);
34
+ export {
35
+ DivinerStageSchema,
36
+ IndexingDivinerConfigSchema,
37
+ IndexingDivinerConfigZod,
38
+ IndexingDivinerSchema,
39
+ asIndexingDivinerConfig,
40
+ isIndexingDivinerConfig,
41
+ toIndexingDivinerConfig
42
+ };
43
+ //# sourceMappingURL=model.mjs.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/model/Config.ts", "../../src/model/Schema.ts", "../../src/model/Stage.ts"],
4
+ "sourcesContent": ["import type { EmptyObject, WithAdditional } from '@ariestools/sdk'\nimport {\n zodAsFactory, zodIsFactory, zodToFactory,\n} from '@ariestools/sdk'\nimport type { DivinerConfig, SearchableStorage } from '@xyo-network/sdk/diviner-model'\nimport { DivinerConfigZod } from '@xyo-network/sdk/diviner-model'\nimport {\n asSchema, type Payload, type Schema,\n} from '@xyo-network/sdk-protocol'\nimport * as z from 'zod/mini'\n\nimport { IndexingDivinerSchema } from './Schema.ts'\nimport type { IndexingDivinerStageConfig } from './Stage.ts'\n\nexport const IndexingDivinerConfigSchema: string & {\n readonly __schema: true\n} = asSchema(`${IndexingDivinerSchema}.config`, true)\nexport type IndexingDivinerConfigSchema = typeof IndexingDivinerConfigSchema\n\nexport const IndexingDivinerConfigZod: z.ZodMiniObject<\n Omit<(typeof DivinerConfigZod)['shape'], 'schema'> & {\n indexStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n indexingDivinerStages: z.ZodMiniOptional<z.ZodMiniCustom<IndexingDivinerStageConfig, IndexingDivinerStageConfig>>\n payloadDivinerLimit: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n pollFrequency: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n schema: z.ZodMiniLiteral<IndexingDivinerConfigSchema>\n stateStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n },\n z.core.$strip\n> = z.extend(DivinerConfigZod, {\n indexStore: z.optional(z.custom<SearchableStorage>()),\n indexingDivinerStages: z.optional(z.custom<IndexingDivinerStageConfig>()),\n payloadDivinerLimit: z.optional(z.number()),\n pollFrequency: z.optional(z.number()),\n schema: z.literal(IndexingDivinerConfigSchema),\n stateStore: z.optional(z.custom<SearchableStorage>()),\n})\n\nexport type IndexingDivinerConfigBase = z.infer<typeof IndexingDivinerConfigZod>\n\nexport const isIndexingDivinerConfig: ReturnType<typeof zodIsFactory<IndexingDivinerConfigBase>> = zodIsFactory(IndexingDivinerConfigZod)\nexport const asIndexingDivinerConfig: ReturnType<typeof zodAsFactory<IndexingDivinerConfigBase>>\n = zodAsFactory(IndexingDivinerConfigZod, 'asIndexingDivinerConfig')\nexport const toIndexingDivinerConfig: ReturnType<typeof zodToFactory<IndexingDivinerConfigBase>>\n = zodToFactory(IndexingDivinerConfigZod, 'toIndexingDivinerConfig')\n\nexport type IndexingDivinerConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = DivinerConfig<\n WithAdditional<\n {\n /**\n * Where the diviner should store it's index\n */\n indexStore?: SearchableStorage\n /**\n * Config section for name/address of individual diviner stages\n */\n indexingDivinerStages?: IndexingDivinerStageConfig\n /**\n * The maximum number of payloads to index at a time\n */\n payloadDivinerLimit?: number\n /**\n * How often to poll for new payloads to index\n */\n pollFrequency?: number\n /**\n * The schema for the Diviner config\n */\n schema: IndexingDivinerConfigSchema\n /**\n * Where the diviner should persist its internal state\n */\n stateStore?: SearchableStorage\n },\n TConfig\n >,\n TSchema\n>\n", "import { asSchema } from '@xyo-network/sdk-protocol'\n\nexport const IndexingDivinerSchema: 'network.xyo.diviner.indexing' & {\n readonly __schema: true\n} = asSchema('network.xyo.diviner.indexing', true)\nexport type ImageThumbnailDivinerSchema = typeof IndexingDivinerSchema\n", "export const DivinerStageSchema: 'network.xyo.diviner.stage' & {\n readonly __schema: true\n} = asSchema('network.xyo.diviner.stage', true)\nexport type DivinerStageSchema = typeof DivinerStageSchema\n\nimport type { ModuleIdentifier } from '@xyo-network/sdk/module-model'\nimport { asSchema } from '@xyo-network/sdk-protocol'\n\n/**\n * The diviners for each stage of an indexing diviner\n */\nexport type IndexingDivinerStage\n /**\n * Transforms a diviner query into an index query\n */\n = | 'divinerQueryToIndexQueryDiviner'\n /**\n * Transforms the index candidates into indexes\n */\n | 'indexCandidateToIndexDiviner'\n /**\n * Transforms an index query response into a diviner divine query response\n */\n | 'indexQueryResponseToDivinerQueryResponseDiviner'\n /**\n * Uses the current state to determine the next batch of index candidates\n */\n | 'stateToIndexCandidateDiviner'\n\n/**\n * Config section for declaring each indexing diviner stage\n */\nexport type IndexingDivinerStageConfig = Record<IndexingDivinerStage, ModuleIdentifier>\n"],
5
+ "mappings": ";AACA;AAAA,EACE;AAAA,EAAc;AAAA,EAAc;AAAA,OACvB;AAEP,SAAS,wBAAwB;AACjC;AAAA,EACE,YAAAA;AAAA,OACK;AACP,YAAY,OAAO;;;ACTnB,SAAS,gBAAgB;AAElB,IAAM,wBAET,SAAS,gCAAgC,IAAI;;;ADU1C,IAAM,8BAETC,UAAS,GAAG,qBAAqB,WAAW,IAAI;AAG7C,IAAM,2BAUP,SAAO,kBAAkB;AAAA,EAC7B,YAAc,WAAW,SAA0B,CAAC;AAAA,EACpD,uBAAyB,WAAW,SAAmC,CAAC;AAAA,EACxE,qBAAuB,WAAW,SAAO,CAAC;AAAA,EAC1C,eAAiB,WAAW,SAAO,CAAC;AAAA,EACpC,QAAU,UAAQ,2BAA2B;AAAA,EAC7C,YAAc,WAAW,SAA0B,CAAC;AACtD,CAAC;AAIM,IAAM,0BAAsF,aAAa,wBAAwB;AACjI,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;AAC7D,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;;;AEtCpE,SAAS,YAAAC,iBAAgB;AANlB,IAAM,qBAETA,UAAS,6BAA6B,IAAI;",
6
+ "names": ["asSchema", "asSchema", "asSchema"]
7
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@xyo-network/diviner-indexing",
3
- "version": "7.2.3",
3
+ "version": "7.3.0",
4
4
  "description": "Primary SDK for using XYO Protocol 2.0",
5
5
  "homepage": "https://xyo.network",
6
6
  "bugs": {
@@ -24,6 +24,14 @@
24
24
  "types": "./dist/neutral/index.d.ts",
25
25
  "default": "./dist/neutral/index.mjs"
26
26
  },
27
+ "./model": {
28
+ "types": "./dist/neutral/model.d.ts",
29
+ "default": "./dist/neutral/model.mjs"
30
+ },
31
+ "./memory": {
32
+ "types": "./dist/neutral/memory.d.ts",
33
+ "default": "./dist/neutral/memory.mjs"
34
+ },
27
35
  "./package.json": "./package.json"
28
36
  },
29
37
  "files": [
@@ -34,21 +42,25 @@
34
42
  "README.md"
35
43
  ],
36
44
  "dependencies": {
37
- "@xyo-network/diviner-indexing-memory": "~7.2.3",
38
- "@xyo-network/diviner-temporal-indexing": "~7.2.3",
39
- "@xyo-network/diviner-indexing-model": "~7.2.3"
45
+ "@xyo-network/sdk": "~7.3.0"
40
46
  },
41
47
  "devDependencies": {
42
- "@ariestools/sdk": "~8.1.6",
43
- "@ariestools/toolchain": "~8.7.26",
44
- "@ariestools/tsconfig": "~8.7.26",
48
+ "@ariestools/sdk": "~8.1.9",
49
+ "@ariestools/toolchain": "~9.1.1",
50
+ "@ariestools/tsconfig": "~9.1.1",
45
51
  "@opentelemetry/api": "~1.9.1",
46
- "@xyo-network/sdk-protocol": "~7.2.3",
47
- "eslint": "~10.8.0",
52
+ "@xyo-network/sdk-protocol": "~7.3.1",
53
+ "eslint": "~10.9.0",
48
54
  "eslint-import-resolver-typescript": "~4.4.5",
55
+ "tslib": "~2.8.1",
49
56
  "typescript": "~6.0.3",
50
57
  "zod": "~4.4.3"
51
58
  },
59
+ "peerDependencies": {
60
+ "@ariestools/sdk": "^8.1",
61
+ "@xyo-network/sdk-protocol": "^7.3",
62
+ "zod": "^4.4"
63
+ },
52
64
  "engines": {
53
65
  "node": "^24"
54
66
  },