@xyo-network/diviner-indexing 7.4.1 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/neutral/index.mjs
CHANGED
|
@@ -112,14 +112,15 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
112
112
|
if (nextState.state.cursor === this._lastState?.state.cursor) return;
|
|
113
113
|
this._lastState = nextState;
|
|
114
114
|
const archivist = await this.getArchivistForStore("stateStore");
|
|
115
|
-
const
|
|
115
|
+
const boundWitnessBuilder = new BoundWitnessBuilder();
|
|
116
|
+
const [bw] = await boundWitnessBuilder.payload(nextState).signer(this.account).build();
|
|
116
117
|
await archivist.insert([bw, nextState]);
|
|
117
118
|
}
|
|
118
119
|
async divineHandler(payloads = []) {
|
|
119
120
|
const indexPayloadDiviner = await this.getPayloadDivinerForStore("indexStore");
|
|
120
121
|
const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage("divinerQueryToIndexQueryDiviner");
|
|
121
122
|
const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage("indexQueryResponseToDivinerQueryResponseDiviner");
|
|
122
|
-
const
|
|
123
|
+
const allResults = await Promise.all(
|
|
123
124
|
payloads.map(async (payload) => {
|
|
124
125
|
const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload]);
|
|
125
126
|
const indexedResults = await indexPayloadDiviner.divine(indexQuery);
|
|
@@ -128,7 +129,8 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
128
129
|
);
|
|
129
130
|
return response.flat();
|
|
130
131
|
})
|
|
131
|
-
)
|
|
132
|
+
);
|
|
133
|
+
const results = allResults.flat();
|
|
132
134
|
return results;
|
|
133
135
|
}
|
|
134
136
|
/**
|
|
@@ -179,11 +181,12 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
179
181
|
* preemptions, reboots, etc.
|
|
180
182
|
*/
|
|
181
183
|
async retrieveState() {
|
|
182
|
-
const accountAddress = this.account.address;
|
|
183
184
|
if (this._lastState) return this._lastState;
|
|
185
|
+
const accountAddress = this.account.address;
|
|
184
186
|
let hash = "";
|
|
185
187
|
const diviner = await this.getBoundWitnessDivinerForStore("stateStore");
|
|
186
|
-
const
|
|
188
|
+
const payloadBuilder = new PayloadBuilder({ schema: BoundWitnessDivinerQuerySchema });
|
|
189
|
+
const query = payloadBuilder.fields({
|
|
187
190
|
address: accountAddress,
|
|
188
191
|
limit: 1,
|
|
189
192
|
cursor: SequenceConstants.minLocalSequence,
|
|
@@ -194,15 +197,20 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
194
197
|
if (boundWitnesses.length > 0) {
|
|
195
198
|
const boundWitness = boundWitnesses[0];
|
|
196
199
|
if (isBoundWitness(boundWitness)) {
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
200
|
+
const matchingAddresses = boundWitness.addresses.map((address, index) => ({ address, index })).filter(({ address }) => address === accountAddress);
|
|
201
|
+
let foundHash = "";
|
|
202
|
+
for (const curr of matchingAddresses) {
|
|
203
|
+
if (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema) {
|
|
204
|
+
foundHash = boundWitness.payload_hashes[curr?.index];
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
hash = foundHash;
|
|
201
208
|
}
|
|
202
209
|
}
|
|
203
210
|
if (hash !== "") {
|
|
204
211
|
const archivist = await this.getArchivistForStore("stateStore");
|
|
205
|
-
const
|
|
212
|
+
const storedPayloads = await archivist.get([hash]);
|
|
213
|
+
const payload = storedPayloads.find(isModuleState);
|
|
206
214
|
if (payload) {
|
|
207
215
|
return payload;
|
|
208
216
|
}
|
|
@@ -232,7 +240,7 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
232
240
|
} catch (e) {
|
|
233
241
|
console.log(e);
|
|
234
242
|
} finally {
|
|
235
|
-
if (this._pollId) clearTimeoutEx(this._pollId);
|
|
243
|
+
if (this._pollId !== void 0 && this._pollId !== "") clearTimeoutEx(this._pollId);
|
|
236
244
|
this._pollId = void 0;
|
|
237
245
|
this.poll();
|
|
238
246
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/memory/Diviner.ts", "../../src/model/Config.ts", "../../src/model/Schema.ts", "../../src/model/Stage.ts"],
|
|
4
|
-
"sourcesContent": ["import type { Hash } from '@ariestools/sdk'\nimport {\n assertEx, clearTimeoutEx, isString, setTimeoutEx,\n} from '@ariestools/sdk'\nimport { ArchivistWrapper } from '@xyo-network/sdk/archivist-wrapper'\nimport { AbstractDiviner } from '@xyo-network/sdk/diviner-abstract'\nimport type { BoundWitnessDivinerQueryPayload } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport type {\n DivinerConfig, DivinerInstance, DivinerModule, DivinerModuleEventData, DivinerParams,\n} from '@xyo-network/sdk/diviner-model'\nimport { asDivinerInstance } from '@xyo-network/sdk/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/sdk/diviner-wrapper'\nimport type { AnyConfigSchema, ModuleState } from '@xyo-network/sdk/module-model'\nimport {\n creatableModule, isModuleState, ModuleStateSchema,\n} from '@xyo-network/sdk/module-model'\nimport type {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/sdk-protocol'\nimport {\n BoundWitnessBuilder, isBoundWitness, PayloadBuilder, SequenceConstants,\n} from '@xyo-network/sdk-protocol'\n\nimport type {\n IndexingDivinerConfig,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '../model/index.ts'\nimport { IndexingDivinerConfigSchema } from '../model/index.ts'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = true\n static override readonly configSchemas: Schema[] = [...super.configSchemas, IndexingDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = IndexingDivinerConfigSchema\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit(): number {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency(): number {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter(x => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void> {\n // Don't commit state if no state has changed\n if (nextState.state.cursor === this._lastState?.state.cursor) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).signer(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map(indexedResult => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore): Promise<ArchivistWrapper> {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage): Promise<DivinerInstance> {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n const accountAddress = this.account.address\n if (this._lastState) return this._lastState\n let hash = '' as Hash\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: accountAddress,\n limit: 1,\n cursor: SequenceConstants.minLocalSequence,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === accountAddress)\n\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '' as Hash,\n )\n }\n }\n\n // If we able to located the last state\n if (hash !== '') {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload as WithStorageMeta<ModuleState<IndexingDivinerState>>\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<void> {\n await super.startHandler()\n this.poll()\n }\n\n protected override async stopHandler(_timeout?: number): Promise<void> {\n if (isString(this._pollId)) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n // eslint-disable-next-line @typescript-eslint/no-misused-promises\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n", "import type { EmptyObject, WithAdditional } from '@ariestools/sdk'\nimport {\n zodAsFactory, zodIsFactory, zodToFactory,\n} from '@ariestools/sdk'\nimport type { DivinerConfig, SearchableStorage } from '@xyo-network/sdk/diviner-model'\nimport { DivinerConfigZod } from '@xyo-network/sdk/diviner-model'\nimport {\n asSchema, type Payload, type Schema,\n} from '@xyo-network/sdk-protocol'\nimport * as z from 'zod/mini'\n\nimport { IndexingDivinerSchema } from './Schema.ts'\nimport type { IndexingDivinerStageConfig } from './Stage.ts'\n\nexport const IndexingDivinerConfigSchema: string & {\n readonly __schema: true\n} = asSchema(`${IndexingDivinerSchema}.config`, true)\nexport type IndexingDivinerConfigSchema = typeof IndexingDivinerConfigSchema\n\nexport const IndexingDivinerConfigZod: z.ZodMiniObject<\n Omit<(typeof DivinerConfigZod)['shape'], 'schema'> & {\n indexStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n indexingDivinerStages: z.ZodMiniOptional<z.ZodMiniCustom<IndexingDivinerStageConfig, IndexingDivinerStageConfig>>\n payloadDivinerLimit: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n pollFrequency: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n schema: z.ZodMiniLiteral<IndexingDivinerConfigSchema>\n stateStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n },\n z.core.$strip\n> = z.extend(DivinerConfigZod, {\n indexStore: z.optional(z.custom<SearchableStorage>()),\n indexingDivinerStages: z.optional(z.custom<IndexingDivinerStageConfig>()),\n payloadDivinerLimit: z.optional(z.number()),\n pollFrequency: z.optional(z.number()),\n schema: z.literal(IndexingDivinerConfigSchema),\n stateStore: z.optional(z.custom<SearchableStorage>()),\n})\n\nexport type IndexingDivinerConfigBase = z.infer<typeof IndexingDivinerConfigZod>\n\nexport const isIndexingDivinerConfig: ReturnType<typeof zodIsFactory<IndexingDivinerConfigBase>> = zodIsFactory(IndexingDivinerConfigZod)\nexport const asIndexingDivinerConfig: ReturnType<typeof zodAsFactory<IndexingDivinerConfigBase>>\n = zodAsFactory(IndexingDivinerConfigZod, 'asIndexingDivinerConfig')\nexport const toIndexingDivinerConfig: ReturnType<typeof zodToFactory<IndexingDivinerConfigBase>>\n = zodToFactory(IndexingDivinerConfigZod, 'toIndexingDivinerConfig')\n\nexport type IndexingDivinerConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = DivinerConfig<\n WithAdditional<\n {\n /**\n * Where the diviner should store it's index\n */\n indexStore?: SearchableStorage\n /**\n * Config section for name/address of individual diviner stages\n */\n indexingDivinerStages?: IndexingDivinerStageConfig\n /**\n * The maximum number of payloads to index at a time\n */\n payloadDivinerLimit?: number\n /**\n * How often to poll for new payloads to index\n */\n pollFrequency?: number\n /**\n * The schema for the Diviner config\n */\n schema: IndexingDivinerConfigSchema\n /**\n * Where the diviner should persist its internal state\n */\n stateStore?: SearchableStorage\n },\n TConfig\n >,\n TSchema\n>\n", "import { asSchema } from '@xyo-network/sdk-protocol'\n\nexport const IndexingDivinerSchema: 'network.xyo.diviner.indexing' & {\n readonly __schema: true\n} = asSchema('network.xyo.diviner.indexing', true)\nexport type ImageThumbnailDivinerSchema = typeof IndexingDivinerSchema\n", "export const DivinerStageSchema: 'network.xyo.diviner.stage' & {\n readonly __schema: true\n} = asSchema('network.xyo.diviner.stage', true)\nexport type DivinerStageSchema = typeof DivinerStageSchema\n\nimport type { ModuleIdentifier } from '@xyo-network/sdk/module-model'\nimport { asSchema } from '@xyo-network/sdk-protocol'\n\n/**\n * The diviners for each stage of an indexing diviner\n */\nexport type IndexingDivinerStage\n /**\n * Transforms a diviner query into an index query\n */\n = | 'divinerQueryToIndexQueryDiviner'\n /**\n * Transforms the index candidates into indexes\n */\n | 'indexCandidateToIndexDiviner'\n /**\n * Transforms an index query response into a diviner divine query response\n */\n | 'indexQueryResponseToDivinerQueryResponseDiviner'\n /**\n * Uses the current state to determine the next batch of index candidates\n */\n | 'stateToIndexCandidateDiviner'\n\n/**\n * Config section for declaring each indexing diviner stage\n */\nexport type IndexingDivinerStageConfig = Record<IndexingDivinerStage, ModuleIdentifier>\n"],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;AACA;AAAA,EACE;AAAA,EAAU;AAAA,EAAgB;AAAA,EAAU;AAAA,OAC/B;AACP,SAAS,wBAAwB;AACjC,SAAS,uBAAuB;AAEhC,SAAS,sCAAsC;AAI/C,SAAS,yBAAyB;AAClC,SAAS,sBAAsB;AAE/B;AAAA,EACE;AAAA,EAAiB;AAAA,EAAe;AAAA,OAC3B;AAIP;AAAA,EACE;AAAA,EAAqB;AAAA,EAAgB;AAAA,EAAgB;AAAA,OAChD;;;ACrBP;AAAA,EACE;AAAA,EAAc;AAAA,EAAc;AAAA,OACvB;AAEP,SAAS,wBAAwB;AACjC;AAAA,EACE,YAAAA;AAAA,OACK;AACP,YAAY,OAAO;;;ACTnB,SAAS,gBAAgB;AAElB,IAAM,wBAET,SAAS,gCAAgC,IAAI;;;ADU1C,IAAM,8BAETC,UAAS,GAAG,qBAAqB,WAAW,IAAI;AAG7C,IAAM,2BAUP,SAAO,kBAAkB;AAAA,EAC7B,YAAc,WAAW,SAA0B,CAAC;AAAA,EACpD,uBAAyB,WAAW,SAAmC,CAAC;AAAA,EACxE,qBAAuB,WAAW,SAAO,CAAC;AAAA,EAC1C,eAAiB,WAAW,SAAO,CAAC;AAAA,EACpC,QAAU,UAAQ,2BAA2B;AAAA,EAC7C,YAAc,WAAW,SAA0B,CAAC;AACtD,CAAC;AAIM,IAAM,0BAAsF,aAAa,wBAAwB;AACjI,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;AAC7D,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;;;AEtCpE,SAAS,YAAAC,iBAAgB;AANlB,IAAM,qBAETA,UAAS,6BAA6B,IAAI;;;AHkC9C,IAAM,aAAa;AAGZ,IAAM,kBAAN,cASG,gBAAgD;AAAA,EAKhD;AAAA,EACA;AAAA,EAER,IAAI,sBAA8B;AAChC,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAwB;AAC1B,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,OAAK,CAAC,cAAc,CAAC,CAAC;AAE7D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA6D;AAEvF,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM,OAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,
|
|
4
|
+
"sourcesContent": ["import type { Hash } from '@ariestools/sdk'\nimport {\n assertEx, clearTimeoutEx, isString, setTimeoutEx,\n} from '@ariestools/sdk'\nimport { ArchivistWrapper } from '@xyo-network/sdk/archivist-wrapper'\nimport { AbstractDiviner } from '@xyo-network/sdk/diviner-abstract'\nimport type { BoundWitnessDivinerQueryPayload } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport type {\n DivinerConfig, DivinerInstance, DivinerModule, DivinerModuleEventData, DivinerParams,\n} from '@xyo-network/sdk/diviner-model'\nimport { asDivinerInstance } from '@xyo-network/sdk/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/sdk/diviner-wrapper'\nimport type { AnyConfigSchema, ModuleState } from '@xyo-network/sdk/module-model'\nimport {\n creatableModule, isModuleState, ModuleStateSchema,\n} from '@xyo-network/sdk/module-model'\nimport type {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/sdk-protocol'\nimport {\n BoundWitnessBuilder, isBoundWitness, PayloadBuilder, SequenceConstants,\n} from '@xyo-network/sdk-protocol'\n\nimport type {\n IndexingDivinerConfig,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '../model/index.ts'\nimport { IndexingDivinerConfigSchema } from '../model/index.ts'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = true\n static override readonly configSchemas: Schema[] = [...super.configSchemas, IndexingDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = IndexingDivinerConfigSchema\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit(): number {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency(): number {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter(x => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void> {\n // Don't commit state if no state has changed\n if (nextState.state.cursor === this._lastState?.state.cursor) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const boundWitnessBuilder = new BoundWitnessBuilder()\n const [bw] = await boundWitnessBuilder.payload(nextState).signer(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const allResults = await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map(indexedResult => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n const results = allResults.flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore): Promise<ArchivistWrapper> {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage): Promise<DivinerInstance> {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n const accountAddress = this.account.address\n let hash = '' as Hash\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const payloadBuilder = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n const query = payloadBuilder\n .fields({\n address: accountAddress,\n limit: 1,\n cursor: SequenceConstants.minLocalSequence,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n const matchingAddresses = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === accountAddress)\n let foundHash = '' as Hash\n for (const curr of matchingAddresses) {\n if (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema) {\n foundHash = boundWitness.payload_hashes[curr?.index]\n }\n }\n hash = foundHash\n }\n }\n\n // If we able to located the last state\n if (hash !== '') {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const storedPayloads = await archivist.get([hash])\n const payload = storedPayloads.find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload as WithStorageMeta<ModuleState<IndexingDivinerState>>\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<void> {\n await super.startHandler()\n this.poll()\n }\n\n protected override async stopHandler(_timeout?: number): Promise<void> {\n if (isString(this._pollId)) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n // eslint-disable-next-line @typescript-eslint/no-misused-promises\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId !== undefined && this._pollId !== '') clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n", "import type { EmptyObject, WithAdditional } from '@ariestools/sdk'\nimport {\n zodAsFactory, zodIsFactory, zodToFactory,\n} from '@ariestools/sdk'\nimport type { DivinerConfig, SearchableStorage } from '@xyo-network/sdk/diviner-model'\nimport { DivinerConfigZod } from '@xyo-network/sdk/diviner-model'\nimport {\n asSchema, type Payload, type Schema,\n} from '@xyo-network/sdk-protocol'\nimport * as z from 'zod/mini'\n\nimport { IndexingDivinerSchema } from './Schema.ts'\nimport type { IndexingDivinerStageConfig } from './Stage.ts'\n\nexport const IndexingDivinerConfigSchema: string & {\n readonly __schema: true\n} = asSchema(`${IndexingDivinerSchema}.config`, true)\nexport type IndexingDivinerConfigSchema = typeof IndexingDivinerConfigSchema\n\nexport const IndexingDivinerConfigZod: z.ZodMiniObject<\n Omit<(typeof DivinerConfigZod)['shape'], 'schema'> & {\n indexStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n indexingDivinerStages: z.ZodMiniOptional<z.ZodMiniCustom<IndexingDivinerStageConfig, IndexingDivinerStageConfig>>\n payloadDivinerLimit: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n pollFrequency: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n schema: z.ZodMiniLiteral<IndexingDivinerConfigSchema>\n stateStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n },\n z.core.$strip\n> = z.extend(DivinerConfigZod, {\n indexStore: z.optional(z.custom<SearchableStorage>()),\n indexingDivinerStages: z.optional(z.custom<IndexingDivinerStageConfig>()),\n payloadDivinerLimit: z.optional(z.number()),\n pollFrequency: z.optional(z.number()),\n schema: z.literal(IndexingDivinerConfigSchema),\n stateStore: z.optional(z.custom<SearchableStorage>()),\n})\n\nexport type IndexingDivinerConfigBase = z.infer<typeof IndexingDivinerConfigZod>\n\nexport const isIndexingDivinerConfig: ReturnType<typeof zodIsFactory<IndexingDivinerConfigBase>> = zodIsFactory(IndexingDivinerConfigZod)\nexport const asIndexingDivinerConfig: ReturnType<typeof zodAsFactory<IndexingDivinerConfigBase>>\n = zodAsFactory(IndexingDivinerConfigZod, 'asIndexingDivinerConfig')\nexport const toIndexingDivinerConfig: ReturnType<typeof zodToFactory<IndexingDivinerConfigBase>>\n = zodToFactory(IndexingDivinerConfigZod, 'toIndexingDivinerConfig')\n\nexport type IndexingDivinerConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = DivinerConfig<\n WithAdditional<\n {\n /**\n * Where the diviner should store it's index\n */\n indexStore?: SearchableStorage\n /**\n * Config section for name/address of individual diviner stages\n */\n indexingDivinerStages?: IndexingDivinerStageConfig\n /**\n * The maximum number of payloads to index at a time\n */\n payloadDivinerLimit?: number\n /**\n * How often to poll for new payloads to index\n */\n pollFrequency?: number\n /**\n * The schema for the Diviner config\n */\n schema: IndexingDivinerConfigSchema\n /**\n * Where the diviner should persist its internal state\n */\n stateStore?: SearchableStorage\n },\n TConfig\n >,\n TSchema\n>\n", "import { asSchema } from '@xyo-network/sdk-protocol'\n\nexport const IndexingDivinerSchema: 'network.xyo.diviner.indexing' & {\n readonly __schema: true\n} = asSchema('network.xyo.diviner.indexing', true)\nexport type ImageThumbnailDivinerSchema = typeof IndexingDivinerSchema\n", "export const DivinerStageSchema: 'network.xyo.diviner.stage' & {\n readonly __schema: true\n} = asSchema('network.xyo.diviner.stage', true)\nexport type DivinerStageSchema = typeof DivinerStageSchema\n\nimport type { ModuleIdentifier } from '@xyo-network/sdk/module-model'\nimport { asSchema } from '@xyo-network/sdk-protocol'\n\n/**\n * The diviners for each stage of an indexing diviner\n */\nexport type IndexingDivinerStage\n /**\n * Transforms a diviner query into an index query\n */\n = | 'divinerQueryToIndexQueryDiviner'\n /**\n * Transforms the index candidates into indexes\n */\n | 'indexCandidateToIndexDiviner'\n /**\n * Transforms an index query response into a diviner divine query response\n */\n | 'indexQueryResponseToDivinerQueryResponseDiviner'\n /**\n * Uses the current state to determine the next batch of index candidates\n */\n | 'stateToIndexCandidateDiviner'\n\n/**\n * Config section for declaring each indexing diviner stage\n */\nexport type IndexingDivinerStageConfig = Record<IndexingDivinerStage, ModuleIdentifier>\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;AACA;AAAA,EACE;AAAA,EAAU;AAAA,EAAgB;AAAA,EAAU;AAAA,OAC/B;AACP,SAAS,wBAAwB;AACjC,SAAS,uBAAuB;AAEhC,SAAS,sCAAsC;AAI/C,SAAS,yBAAyB;AAClC,SAAS,sBAAsB;AAE/B;AAAA,EACE;AAAA,EAAiB;AAAA,EAAe;AAAA,OAC3B;AAIP;AAAA,EACE;AAAA,EAAqB;AAAA,EAAgB;AAAA,EAAgB;AAAA,OAChD;;;ACrBP;AAAA,EACE;AAAA,EAAc;AAAA,EAAc;AAAA,OACvB;AAEP,SAAS,wBAAwB;AACjC;AAAA,EACE,YAAAA;AAAA,OACK;AACP,YAAY,OAAO;;;ACTnB,SAAS,gBAAgB;AAElB,IAAM,wBAET,SAAS,gCAAgC,IAAI;;;ADU1C,IAAM,8BAETC,UAAS,GAAG,qBAAqB,WAAW,IAAI;AAG7C,IAAM,2BAUP,SAAO,kBAAkB;AAAA,EAC7B,YAAc,WAAW,SAA0B,CAAC;AAAA,EACpD,uBAAyB,WAAW,SAAmC,CAAC;AAAA,EACxE,qBAAuB,WAAW,SAAO,CAAC;AAAA,EAC1C,eAAiB,WAAW,SAAO,CAAC;AAAA,EACpC,QAAU,UAAQ,2BAA2B;AAAA,EAC7C,YAAc,WAAW,SAA0B,CAAC;AACtD,CAAC;AAIM,IAAM,0BAAsF,aAAa,wBAAwB;AACjI,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;AAC7D,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;;;AEtCpE,SAAS,YAAAC,iBAAgB;AANlB,IAAM,qBAETA,UAAS,6BAA6B,IAAI;;;AHkC9C,IAAM,aAAa;AAGZ,IAAM,kBAAN,cASG,gBAAgD;AAAA,EAKhD;AAAA,EACA;AAAA,EAER,IAAI,sBAA8B;AAChC,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAwB;AAC1B,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,OAAK,CAAC,cAAc,CAAC,CAAC;AAE7D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA6D;AAEvF,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM,OAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,sBAAsB,IAAI,oBAAoB;AACpD,UAAM,CAAC,EAAE,IAAI,MAAM,oBAAoB,QAAQ,SAAS,EAAE,OAAO,KAAK,OAAO,EAAE,MAAM;AACrF,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,aAAa,MAAM,QAAQ;AAAA,MAC/B,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,mBAAiB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC7H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH;AACA,UAAM,UAAU,WAAW,KAAK;AAEhC,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAA+C;AAClF,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,eAAe,IAAI,GAAG;AACpH,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAqJ;AAClM,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,yBAAyB,IAAI,GAAG;AAC9H,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAA2D;AACjG,UAAM,gBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAqJ;AAC7L,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,oBAAoB,IAAI,GAAG;AACzH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK,WAAY,QAAO,KAAK;AACjC,UAAM,iBAAiB,KAAK,QAAQ;AACpC,QAAI,OAAO;AACX,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,iBAAiB,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC;AACrH,UAAM,QAAQ,eACX,OAAO;AAAA,MACN,SAAS;AAAA,MACT,OAAO;AAAA,MACP,QAAQ,kBAAkB;AAAA,MAC1B,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,cAAM,oBAAoB,aAAa,UACpC,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,cAAc;AACrD,YAAI,YAAY;AAChB,mBAAW,QAAQ,mBAAmB;AACpC,cAAI,aAAa,kBAAkB,MAAM,KAAK,MAAM,mBAAmB;AACrE,wBAAY,aAAa,eAAe,MAAM,KAAK;AAAA,UACrD;AAAA,QACF;AACA,eAAO;AAAA,MACT;AAAA,IACF;AAGA,QAAI,SAAS,IAAI;AAEf,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,iBAAiB,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC;AACjD,YAAM,UAAU,eAAe,KAAK,aAAmC;AACvE,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAA8B;AACrD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAyB,YAAY,UAAkC;AACrE,QAAI,SAAS,KAAK,OAAO,GAAG;AAC1B,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,UAAM,MAAM,YAAY;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AAEb,SAAK,UAAU,aAAa,YAAY;AACtC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK,YAAY,UAAa,KAAK,YAAY,GAAI,gBAAe,KAAK,OAAO;AAClF,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;AA/ME,cAVW,iBAUc,sBAAqB;AAC9C,cAXW,iBAWc,iBAA0B,CAAC,GAAG,6CAAM,kBAAe,2BAA2B;AACvG,cAZW,iBAYc,uBAA8B;AAZ5C,kBAAN;AAAA,EADN,gBAAiC;AAAA,GACrB;",
|
|
6
6
|
"names": ["asSchema", "asSchema", "asSchema"]
|
|
7
7
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/memory/Diviner.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAA;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAA;AAGnE,OAAO,KAAK,EACV,aAAa,EAAE,eAAe,EAAE,aAAa,EAAE,sBAAsB,EAAE,aAAa,EACrF,MAAM,gCAAgC,CAAA;AAEvC,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAA;AACjE,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAA;AAIjF,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAChB,MAAM,2BAA2B,CAAA;AAKlC,OAAO,KAAK,EACV,qBAAqB,EACrB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,mBAAmB,CAAA;AAG1B,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,QAAO;IAClD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAwD;IACxG,gBAAyB,mBAAmB,EAAE,MAAM,CAA8B;IAElF,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED,IAAI,aAAa,IAAI,MAAM,CAE1B;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,OAAO,CAAC,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/memory/Diviner.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAA;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAA;AAGnE,OAAO,KAAK,EACV,aAAa,EAAE,eAAe,EAAE,aAAa,EAAE,sBAAsB,EAAE,aAAa,EACrF,MAAM,gCAAgC,CAAA;AAEvC,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAA;AACjE,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAA;AAIjF,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAChB,MAAM,2BAA2B,CAAA;AAKlC,OAAO,KAAK,EACV,qBAAqB,EACrB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,mBAAmB,CAAA;AAG1B,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAChH,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EACnC,GAAG,EACH,IAAI,CACL,CACD,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,QAAO;IAClD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAwD;IACxG,gBAAyB,mBAAmB,EAAE,MAAM,CAA8B;IAElF,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED,IAAI,aAAa,IAAI,MAAM,CAE1B;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,OAAO,CAAC,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;cAU/D,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAqB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAMnF;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,aAAa,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,EAAE,sBAAsB,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAMnM;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB,GAAG,OAAO,CAAC,eAAe,CAAC;IASlG;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,aAAa,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,EAAE,sBAAsB,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAM9L;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cA8C9D,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;cAK7B,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQtE;;;OAGG;IACH,OAAO,CAAC,IAAI;CAeb"}
|
package/dist/neutral/memory.mjs
CHANGED
|
@@ -108,14 +108,15 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
108
108
|
if (nextState.state.cursor === this._lastState?.state.cursor) return;
|
|
109
109
|
this._lastState = nextState;
|
|
110
110
|
const archivist = await this.getArchivistForStore("stateStore");
|
|
111
|
-
const
|
|
111
|
+
const boundWitnessBuilder = new BoundWitnessBuilder();
|
|
112
|
+
const [bw] = await boundWitnessBuilder.payload(nextState).signer(this.account).build();
|
|
112
113
|
await archivist.insert([bw, nextState]);
|
|
113
114
|
}
|
|
114
115
|
async divineHandler(payloads = []) {
|
|
115
116
|
const indexPayloadDiviner = await this.getPayloadDivinerForStore("indexStore");
|
|
116
117
|
const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage("divinerQueryToIndexQueryDiviner");
|
|
117
118
|
const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage("indexQueryResponseToDivinerQueryResponseDiviner");
|
|
118
|
-
const
|
|
119
|
+
const allResults = await Promise.all(
|
|
119
120
|
payloads.map(async (payload) => {
|
|
120
121
|
const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload]);
|
|
121
122
|
const indexedResults = await indexPayloadDiviner.divine(indexQuery);
|
|
@@ -124,7 +125,8 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
124
125
|
);
|
|
125
126
|
return response.flat();
|
|
126
127
|
})
|
|
127
|
-
)
|
|
128
|
+
);
|
|
129
|
+
const results = allResults.flat();
|
|
128
130
|
return results;
|
|
129
131
|
}
|
|
130
132
|
/**
|
|
@@ -175,11 +177,12 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
175
177
|
* preemptions, reboots, etc.
|
|
176
178
|
*/
|
|
177
179
|
async retrieveState() {
|
|
178
|
-
const accountAddress = this.account.address;
|
|
179
180
|
if (this._lastState) return this._lastState;
|
|
181
|
+
const accountAddress = this.account.address;
|
|
180
182
|
let hash = "";
|
|
181
183
|
const diviner = await this.getBoundWitnessDivinerForStore("stateStore");
|
|
182
|
-
const
|
|
184
|
+
const payloadBuilder = new PayloadBuilder({ schema: BoundWitnessDivinerQuerySchema });
|
|
185
|
+
const query = payloadBuilder.fields({
|
|
183
186
|
address: accountAddress,
|
|
184
187
|
limit: 1,
|
|
185
188
|
cursor: SequenceConstants.minLocalSequence,
|
|
@@ -190,15 +193,20 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
190
193
|
if (boundWitnesses.length > 0) {
|
|
191
194
|
const boundWitness = boundWitnesses[0];
|
|
192
195
|
if (isBoundWitness(boundWitness)) {
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
196
|
+
const matchingAddresses = boundWitness.addresses.map((address, index) => ({ address, index })).filter(({ address }) => address === accountAddress);
|
|
197
|
+
let foundHash = "";
|
|
198
|
+
for (const curr of matchingAddresses) {
|
|
199
|
+
if (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema) {
|
|
200
|
+
foundHash = boundWitness.payload_hashes[curr?.index];
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
hash = foundHash;
|
|
197
204
|
}
|
|
198
205
|
}
|
|
199
206
|
if (hash !== "") {
|
|
200
207
|
const archivist = await this.getArchivistForStore("stateStore");
|
|
201
|
-
const
|
|
208
|
+
const storedPayloads = await archivist.get([hash]);
|
|
209
|
+
const payload = storedPayloads.find(isModuleState);
|
|
202
210
|
if (payload) {
|
|
203
211
|
return payload;
|
|
204
212
|
}
|
|
@@ -228,7 +236,7 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
228
236
|
} catch (e) {
|
|
229
237
|
console.log(e);
|
|
230
238
|
} finally {
|
|
231
|
-
if (this._pollId) clearTimeoutEx(this._pollId);
|
|
239
|
+
if (this._pollId !== void 0 && this._pollId !== "") clearTimeoutEx(this._pollId);
|
|
232
240
|
this._pollId = void 0;
|
|
233
241
|
this.poll();
|
|
234
242
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/memory/Diviner.ts", "../../src/model/Config.ts", "../../src/model/Schema.ts"],
|
|
4
|
-
"sourcesContent": ["import type { Hash } from '@ariestools/sdk'\nimport {\n assertEx, clearTimeoutEx, isString, setTimeoutEx,\n} from '@ariestools/sdk'\nimport { ArchivistWrapper } from '@xyo-network/sdk/archivist-wrapper'\nimport { AbstractDiviner } from '@xyo-network/sdk/diviner-abstract'\nimport type { BoundWitnessDivinerQueryPayload } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport type {\n DivinerConfig, DivinerInstance, DivinerModule, DivinerModuleEventData, DivinerParams,\n} from '@xyo-network/sdk/diviner-model'\nimport { asDivinerInstance } from '@xyo-network/sdk/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/sdk/diviner-wrapper'\nimport type { AnyConfigSchema, ModuleState } from '@xyo-network/sdk/module-model'\nimport {\n creatableModule, isModuleState, ModuleStateSchema,\n} from '@xyo-network/sdk/module-model'\nimport type {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/sdk-protocol'\nimport {\n BoundWitnessBuilder, isBoundWitness, PayloadBuilder, SequenceConstants,\n} from '@xyo-network/sdk-protocol'\n\nimport type {\n IndexingDivinerConfig,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '../model/index.ts'\nimport { IndexingDivinerConfigSchema } from '../model/index.ts'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = true\n static override readonly configSchemas: Schema[] = [...super.configSchemas, IndexingDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = IndexingDivinerConfigSchema\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit(): number {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency(): number {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter(x => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void> {\n // Don't commit state if no state has changed\n if (nextState.state.cursor === this._lastState?.state.cursor) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).signer(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map(indexedResult => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore): Promise<ArchivistWrapper> {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage): Promise<DivinerInstance> {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n const accountAddress = this.account.address\n if (this._lastState) return this._lastState\n let hash = '' as Hash\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: accountAddress,\n limit: 1,\n cursor: SequenceConstants.minLocalSequence,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === accountAddress)\n\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '' as Hash,\n )\n }\n }\n\n // If we able to located the last state\n if (hash !== '') {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload as WithStorageMeta<ModuleState<IndexingDivinerState>>\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<void> {\n await super.startHandler()\n this.poll()\n }\n\n protected override async stopHandler(_timeout?: number): Promise<void> {\n if (isString(this._pollId)) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n // eslint-disable-next-line @typescript-eslint/no-misused-promises\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n", "import type { EmptyObject, WithAdditional } from '@ariestools/sdk'\nimport {\n zodAsFactory, zodIsFactory, zodToFactory,\n} from '@ariestools/sdk'\nimport type { DivinerConfig, SearchableStorage } from '@xyo-network/sdk/diviner-model'\nimport { DivinerConfigZod } from '@xyo-network/sdk/diviner-model'\nimport {\n asSchema, type Payload, type Schema,\n} from '@xyo-network/sdk-protocol'\nimport * as z from 'zod/mini'\n\nimport { IndexingDivinerSchema } from './Schema.ts'\nimport type { IndexingDivinerStageConfig } from './Stage.ts'\n\nexport const IndexingDivinerConfigSchema: string & {\n readonly __schema: true\n} = asSchema(`${IndexingDivinerSchema}.config`, true)\nexport type IndexingDivinerConfigSchema = typeof IndexingDivinerConfigSchema\n\nexport const IndexingDivinerConfigZod: z.ZodMiniObject<\n Omit<(typeof DivinerConfigZod)['shape'], 'schema'> & {\n indexStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n indexingDivinerStages: z.ZodMiniOptional<z.ZodMiniCustom<IndexingDivinerStageConfig, IndexingDivinerStageConfig>>\n payloadDivinerLimit: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n pollFrequency: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n schema: z.ZodMiniLiteral<IndexingDivinerConfigSchema>\n stateStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n },\n z.core.$strip\n> = z.extend(DivinerConfigZod, {\n indexStore: z.optional(z.custom<SearchableStorage>()),\n indexingDivinerStages: z.optional(z.custom<IndexingDivinerStageConfig>()),\n payloadDivinerLimit: z.optional(z.number()),\n pollFrequency: z.optional(z.number()),\n schema: z.literal(IndexingDivinerConfigSchema),\n stateStore: z.optional(z.custom<SearchableStorage>()),\n})\n\nexport type IndexingDivinerConfigBase = z.infer<typeof IndexingDivinerConfigZod>\n\nexport const isIndexingDivinerConfig: ReturnType<typeof zodIsFactory<IndexingDivinerConfigBase>> = zodIsFactory(IndexingDivinerConfigZod)\nexport const asIndexingDivinerConfig: ReturnType<typeof zodAsFactory<IndexingDivinerConfigBase>>\n = zodAsFactory(IndexingDivinerConfigZod, 'asIndexingDivinerConfig')\nexport const toIndexingDivinerConfig: ReturnType<typeof zodToFactory<IndexingDivinerConfigBase>>\n = zodToFactory(IndexingDivinerConfigZod, 'toIndexingDivinerConfig')\n\nexport type IndexingDivinerConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = DivinerConfig<\n WithAdditional<\n {\n /**\n * Where the diviner should store it's index\n */\n indexStore?: SearchableStorage\n /**\n * Config section for name/address of individual diviner stages\n */\n indexingDivinerStages?: IndexingDivinerStageConfig\n /**\n * The maximum number of payloads to index at a time\n */\n payloadDivinerLimit?: number\n /**\n * How often to poll for new payloads to index\n */\n pollFrequency?: number\n /**\n * The schema for the Diviner config\n */\n schema: IndexingDivinerConfigSchema\n /**\n * Where the diviner should persist its internal state\n */\n stateStore?: SearchableStorage\n },\n TConfig\n >,\n TSchema\n>\n", "import { asSchema } from '@xyo-network/sdk-protocol'\n\nexport const IndexingDivinerSchema: 'network.xyo.diviner.indexing' & {\n readonly __schema: true\n} = asSchema('network.xyo.diviner.indexing', true)\nexport type ImageThumbnailDivinerSchema = typeof IndexingDivinerSchema\n"],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;AACA;AAAA,EACE;AAAA,EAAU;AAAA,EAAgB;AAAA,EAAU;AAAA,OAC/B;AACP,SAAS,wBAAwB;AACjC,SAAS,uBAAuB;AAEhC,SAAS,sCAAsC;AAI/C,SAAS,yBAAyB;AAClC,SAAS,sBAAsB;AAE/B;AAAA,EACE;AAAA,EAAiB;AAAA,EAAe;AAAA,OAC3B;AAIP;AAAA,EACE;AAAA,EAAqB;AAAA,EAAgB;AAAA,EAAgB;AAAA,OAChD;;;ACrBP;AAAA,EACE;AAAA,EAAc;AAAA,EAAc;AAAA,OACvB;AAEP,SAAS,wBAAwB;AACjC;AAAA,EACE,YAAAA;AAAA,OACK;AACP,YAAY,OAAO;;;ACTnB,SAAS,gBAAgB;AAElB,IAAM,wBAET,SAAS,gCAAgC,IAAI;;;ADU1C,IAAM,8BAETC,UAAS,GAAG,qBAAqB,WAAW,IAAI;AAG7C,IAAM,2BAUP,SAAO,kBAAkB;AAAA,EAC7B,YAAc,WAAW,SAA0B,CAAC;AAAA,EACpD,uBAAyB,WAAW,SAAmC,CAAC;AAAA,EACxE,qBAAuB,WAAW,SAAO,CAAC;AAAA,EAC1C,eAAiB,WAAW,SAAO,CAAC;AAAA,EACpC,QAAU,UAAQ,2BAA2B;AAAA,EAC7C,YAAc,WAAW,SAA0B,CAAC;AACtD,CAAC;AAIM,IAAM,0BAAsF,aAAa,wBAAwB;AACjI,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;AAC7D,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;;;ADRpE,IAAM,aAAa;AAGZ,IAAM,kBAAN,cASG,gBAAgD;AAAA,EAKhD;AAAA,EACA;AAAA,EAER,IAAI,sBAA8B;AAChC,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAwB;AAC1B,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,OAAK,CAAC,cAAc,CAAC,CAAC;AAE7D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA6D;AAEvF,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM,OAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,
|
|
4
|
+
"sourcesContent": ["import type { Hash } from '@ariestools/sdk'\nimport {\n assertEx, clearTimeoutEx, isString, setTimeoutEx,\n} from '@ariestools/sdk'\nimport { ArchivistWrapper } from '@xyo-network/sdk/archivist-wrapper'\nimport { AbstractDiviner } from '@xyo-network/sdk/diviner-abstract'\nimport type { BoundWitnessDivinerQueryPayload } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/sdk/diviner-boundwitness/model'\nimport type {\n DivinerConfig, DivinerInstance, DivinerModule, DivinerModuleEventData, DivinerParams,\n} from '@xyo-network/sdk/diviner-model'\nimport { asDivinerInstance } from '@xyo-network/sdk/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/sdk/diviner-wrapper'\nimport type { AnyConfigSchema, ModuleState } from '@xyo-network/sdk/module-model'\nimport {\n creatableModule, isModuleState, ModuleStateSchema,\n} from '@xyo-network/sdk/module-model'\nimport type {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/sdk-protocol'\nimport {\n BoundWitnessBuilder, isBoundWitness, PayloadBuilder, SequenceConstants,\n} from '@xyo-network/sdk-protocol'\n\nimport type {\n IndexingDivinerConfig,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '../model/index.ts'\nimport { IndexingDivinerConfigSchema } from '../model/index.ts'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n DivinerInstance<TParams, TIn, TOut>,\n TIn,\n TOut\n >,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = true\n static override readonly configSchemas: Schema[] = [...super.configSchemas, IndexingDivinerConfigSchema]\n static override readonly defaultConfigSchema: Schema = IndexingDivinerConfigSchema\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit(): number {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency(): number {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter(x => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>): Promise<void> {\n // Don't commit state if no state has changed\n if (nextState.state.cursor === this._lastState?.state.cursor) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const boundWitnessBuilder = new BoundWitnessBuilder()\n const [bw] = await boundWitnessBuilder.payload(nextState).signer(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const allResults = await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map(indexedResult => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n const results = allResults.flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore): Promise<ArchivistWrapper> {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage): Promise<DivinerInstance> {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore): Promise<DivinerWrapper<DivinerModule<DivinerParams<AnyConfigSchema<DivinerConfig>>, DivinerModuleEventData>, Payload, Payload>> {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n const accountAddress = this.account.address\n let hash = '' as Hash\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const payloadBuilder = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n const query = payloadBuilder\n .fields({\n address: accountAddress,\n limit: 1,\n cursor: SequenceConstants.minLocalSequence,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n const matchingAddresses = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === accountAddress)\n let foundHash = '' as Hash\n for (const curr of matchingAddresses) {\n if (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema) {\n foundHash = boundWitness.payload_hashes[curr?.index]\n }\n }\n hash = foundHash\n }\n }\n\n // If we able to located the last state\n if (hash !== '') {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const storedPayloads = await archivist.get([hash])\n const payload = storedPayloads.find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload as WithStorageMeta<ModuleState<IndexingDivinerState>>\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<void> {\n await super.startHandler()\n this.poll()\n }\n\n protected override async stopHandler(_timeout?: number): Promise<void> {\n if (isString(this._pollId)) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n // eslint-disable-next-line @typescript-eslint/no-misused-promises\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId !== undefined && this._pollId !== '') clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n", "import type { EmptyObject, WithAdditional } from '@ariestools/sdk'\nimport {\n zodAsFactory, zodIsFactory, zodToFactory,\n} from '@ariestools/sdk'\nimport type { DivinerConfig, SearchableStorage } from '@xyo-network/sdk/diviner-model'\nimport { DivinerConfigZod } from '@xyo-network/sdk/diviner-model'\nimport {\n asSchema, type Payload, type Schema,\n} from '@xyo-network/sdk-protocol'\nimport * as z from 'zod/mini'\n\nimport { IndexingDivinerSchema } from './Schema.ts'\nimport type { IndexingDivinerStageConfig } from './Stage.ts'\n\nexport const IndexingDivinerConfigSchema: string & {\n readonly __schema: true\n} = asSchema(`${IndexingDivinerSchema}.config`, true)\nexport type IndexingDivinerConfigSchema = typeof IndexingDivinerConfigSchema\n\nexport const IndexingDivinerConfigZod: z.ZodMiniObject<\n Omit<(typeof DivinerConfigZod)['shape'], 'schema'> & {\n indexStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n indexingDivinerStages: z.ZodMiniOptional<z.ZodMiniCustom<IndexingDivinerStageConfig, IndexingDivinerStageConfig>>\n payloadDivinerLimit: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n pollFrequency: z.ZodMiniOptional<z.ZodMiniNumber<number>>\n schema: z.ZodMiniLiteral<IndexingDivinerConfigSchema>\n stateStore: z.ZodMiniOptional<z.ZodMiniCustom<SearchableStorage, SearchableStorage>>\n },\n z.core.$strip\n> = z.extend(DivinerConfigZod, {\n indexStore: z.optional(z.custom<SearchableStorage>()),\n indexingDivinerStages: z.optional(z.custom<IndexingDivinerStageConfig>()),\n payloadDivinerLimit: z.optional(z.number()),\n pollFrequency: z.optional(z.number()),\n schema: z.literal(IndexingDivinerConfigSchema),\n stateStore: z.optional(z.custom<SearchableStorage>()),\n})\n\nexport type IndexingDivinerConfigBase = z.infer<typeof IndexingDivinerConfigZod>\n\nexport const isIndexingDivinerConfig: ReturnType<typeof zodIsFactory<IndexingDivinerConfigBase>> = zodIsFactory(IndexingDivinerConfigZod)\nexport const asIndexingDivinerConfig: ReturnType<typeof zodAsFactory<IndexingDivinerConfigBase>>\n = zodAsFactory(IndexingDivinerConfigZod, 'asIndexingDivinerConfig')\nexport const toIndexingDivinerConfig: ReturnType<typeof zodToFactory<IndexingDivinerConfigBase>>\n = zodToFactory(IndexingDivinerConfigZod, 'toIndexingDivinerConfig')\n\nexport type IndexingDivinerConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = DivinerConfig<\n WithAdditional<\n {\n /**\n * Where the diviner should store it's index\n */\n indexStore?: SearchableStorage\n /**\n * Config section for name/address of individual diviner stages\n */\n indexingDivinerStages?: IndexingDivinerStageConfig\n /**\n * The maximum number of payloads to index at a time\n */\n payloadDivinerLimit?: number\n /**\n * How often to poll for new payloads to index\n */\n pollFrequency?: number\n /**\n * The schema for the Diviner config\n */\n schema: IndexingDivinerConfigSchema\n /**\n * Where the diviner should persist its internal state\n */\n stateStore?: SearchableStorage\n },\n TConfig\n >,\n TSchema\n>\n", "import { asSchema } from '@xyo-network/sdk-protocol'\n\nexport const IndexingDivinerSchema: 'network.xyo.diviner.indexing' & {\n readonly __schema: true\n} = asSchema('network.xyo.diviner.indexing', true)\nexport type ImageThumbnailDivinerSchema = typeof IndexingDivinerSchema\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;AACA;AAAA,EACE;AAAA,EAAU;AAAA,EAAgB;AAAA,EAAU;AAAA,OAC/B;AACP,SAAS,wBAAwB;AACjC,SAAS,uBAAuB;AAEhC,SAAS,sCAAsC;AAI/C,SAAS,yBAAyB;AAClC,SAAS,sBAAsB;AAE/B;AAAA,EACE;AAAA,EAAiB;AAAA,EAAe;AAAA,OAC3B;AAIP;AAAA,EACE;AAAA,EAAqB;AAAA,EAAgB;AAAA,EAAgB;AAAA,OAChD;;;ACrBP;AAAA,EACE;AAAA,EAAc;AAAA,EAAc;AAAA,OACvB;AAEP,SAAS,wBAAwB;AACjC;AAAA,EACE,YAAAA;AAAA,OACK;AACP,YAAY,OAAO;;;ACTnB,SAAS,gBAAgB;AAElB,IAAM,wBAET,SAAS,gCAAgC,IAAI;;;ADU1C,IAAM,8BAETC,UAAS,GAAG,qBAAqB,WAAW,IAAI;AAG7C,IAAM,2BAUP,SAAO,kBAAkB;AAAA,EAC7B,YAAc,WAAW,SAA0B,CAAC;AAAA,EACpD,uBAAyB,WAAW,SAAmC,CAAC;AAAA,EACxE,qBAAuB,WAAW,SAAO,CAAC;AAAA,EAC1C,eAAiB,WAAW,SAAO,CAAC;AAAA,EACpC,QAAU,UAAQ,2BAA2B;AAAA,EAC7C,YAAc,WAAW,SAA0B,CAAC;AACtD,CAAC;AAIM,IAAM,0BAAsF,aAAa,wBAAwB;AACjI,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;AAC7D,IAAM,0BACT,aAAa,0BAA0B,yBAAyB;;;ADRpE,IAAM,aAAa;AAGZ,IAAM,kBAAN,cASG,gBAAgD;AAAA,EAKhD;AAAA,EACA;AAAA,EAER,IAAI,sBAA8B;AAChC,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAwB;AAC1B,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,OAAK,CAAC,cAAc,CAAC,CAAC;AAE7D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA6D;AAEvF,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM,OAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,sBAAsB,IAAI,oBAAoB;AACpD,UAAM,CAAC,EAAE,IAAI,MAAM,oBAAoB,QAAQ,SAAS,EAAE,OAAO,KAAK,OAAO,EAAE,MAAM;AACrF,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,aAAa,MAAM,QAAQ;AAAA,MAC/B,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,mBAAiB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC7H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH;AACA,UAAM,UAAU,WAAW,KAAK;AAEhC,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAA+C;AAClF,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,eAAe,IAAI,GAAG;AACpH,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAqJ;AAClM,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,yBAAyB,IAAI,GAAG;AAC9H,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAA2D;AACjG,UAAM,gBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAqJ;AAC7L,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,oBAAoB,IAAI,GAAG;AACzH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK,WAAY,QAAO,KAAK;AACjC,UAAM,iBAAiB,KAAK,QAAQ;AACpC,QAAI,OAAO;AACX,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,iBAAiB,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC;AACrH,UAAM,QAAQ,eACX,OAAO;AAAA,MACN,SAAS;AAAA,MACT,OAAO;AAAA,MACP,QAAQ,kBAAkB;AAAA,MAC1B,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,cAAM,oBAAoB,aAAa,UACpC,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,cAAc;AACrD,YAAI,YAAY;AAChB,mBAAW,QAAQ,mBAAmB;AACpC,cAAI,aAAa,kBAAkB,MAAM,KAAK,MAAM,mBAAmB;AACrE,wBAAY,aAAa,eAAe,MAAM,KAAK;AAAA,UACrD;AAAA,QACF;AACA,eAAO;AAAA,MACT;AAAA,IACF;AAGA,QAAI,SAAS,IAAI;AAEf,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,iBAAiB,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC;AACjD,YAAM,UAAU,eAAe,KAAK,aAAmC;AACvE,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAA8B;AACrD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAyB,YAAY,UAAkC;AACrE,QAAI,SAAS,KAAK,OAAO,GAAG;AAC1B,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,UAAM,MAAM,YAAY;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AAEb,SAAK,UAAU,aAAa,YAAY;AACtC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK,YAAY,UAAa,KAAK,YAAY,GAAI,gBAAe,KAAK,OAAO;AAClF,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;AA/ME,cAVW,iBAUc,sBAAqB;AAC9C,cAXW,iBAWc,iBAA0B,CAAC,GAAG,6CAAM,kBAAe,2BAA2B;AACvG,cAZW,iBAYc,uBAA8B;AAZ5C,kBAAN;AAAA,EADN,gBAAiC;AAAA,GACrB;",
|
|
6
6
|
"names": ["asSchema", "asSchema"]
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xyo-network/diviner-indexing",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "8.0.0",
|
|
4
4
|
"description": "Primary SDK for using XYO Protocol 2.0",
|
|
5
5
|
"homepage": "https://xyo.network",
|
|
6
6
|
"bugs": {
|
|
@@ -42,27 +42,27 @@
|
|
|
42
42
|
"README.md"
|
|
43
43
|
],
|
|
44
44
|
"dependencies": {
|
|
45
|
-
"@xyo-network/sdk": "~
|
|
45
|
+
"@xyo-network/sdk": "~8.0.0"
|
|
46
46
|
},
|
|
47
47
|
"devDependencies": {
|
|
48
|
-
"@ariestools/sdk": "~
|
|
49
|
-
"@ariestools/toolchain": "~
|
|
50
|
-
"@ariestools/tsconfig": "~
|
|
48
|
+
"@ariestools/sdk": "~9.0.0",
|
|
49
|
+
"@ariestools/toolchain": "~10.0.7",
|
|
50
|
+
"@ariestools/tsconfig": "~10.0.7",
|
|
51
51
|
"@opentelemetry/api": "~1.9.1",
|
|
52
|
-
"@xyo-network/sdk-protocol": "~
|
|
52
|
+
"@xyo-network/sdk-protocol": "~8.0.0",
|
|
53
53
|
"eslint": "~10.10.0",
|
|
54
54
|
"eslint-import-resolver-typescript": "~4.4.5",
|
|
55
55
|
"tslib": "~2.8.1",
|
|
56
56
|
"typescript": "~6.0.3",
|
|
57
|
-
"zod": "~4.5
|
|
57
|
+
"zod": "~4.6.5"
|
|
58
58
|
},
|
|
59
59
|
"peerDependencies": {
|
|
60
|
-
"@ariestools/sdk": "^
|
|
61
|
-
"@xyo-network/sdk-protocol": "^
|
|
62
|
-
"zod": "^4.
|
|
60
|
+
"@ariestools/sdk": "^9.0",
|
|
61
|
+
"@xyo-network/sdk-protocol": "^8.0",
|
|
62
|
+
"zod": "^4.6"
|
|
63
63
|
},
|
|
64
64
|
"engines": {
|
|
65
|
-
"node": "^
|
|
65
|
+
"node": "^26"
|
|
66
66
|
},
|
|
67
67
|
"publishConfig": {
|
|
68
68
|
"access": "public"
|