@xyo-network/diviner-temporal-indexing-memory 3.6.0-rc.3 → 3.6.0-rc.4

package/dist/browser/StateToIndexCandidateDiviner/Diviner.d.ts

@@ -8,7 +8,7 @@ import type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model';
 import type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model';
 import { DivinerWrapper } from '@xyo-network/diviner-wrapper';
 import type { Labels, ModuleState } from '@xyo-network/module-model';
-import type { Payload, Schema } from '@xyo-network/payload-model';
+import type { Payload, Schema, WithStorageMeta } from '@xyo-network/payload-model';
 import type { TimeStamp } from '@xyo-network/witness-timestamp';
 /**
  * All Payload types involved in index candidates for indexing
@@ -101,7 +101,7 @@ export declare class TemporalIndexingDivinerStateToIndexCandidateDiviner<TParams
         schema: BoundWitnessDivinerQuerySchema;
     } & import("@xyo-network/diviner-boundwitness-model").BoundWitnessDivinerPredicate & import("@xyo-network/payload-model").QueryFields, "schema"> & {
         schema: "network.xyo.diviner.boundwitness.query";
-    }, BoundWitness> | undefined>;
+    }, WithStorageMeta<BoundWitness>> | undefined>;
     protected getPayloadsInBoundWitness(bw: BoundWitness, archivist: ArchivistInstance): Promise<IndexCandidate[] | undefined>;
 }
 //# sourceMappingURL=Diviner.d.ts.map

package/dist/browser/StateToIndexCandidateDiviner/Diviner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/StateToIndexCandidateDiviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAA;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAEnE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4CAA4C,CAAA;AAErF,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAA;AACxF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAA;AAC/E,OAAO,KAAK,EAAE,yDAAyD,EAAE,MAAM,8CAA8C,CAAA;AAE7H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EACV,MAAM,EAAoB,WAAW,EACtC,MAAM,2BAA2B,CAAA;AAGlC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAA;AAEjE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAA;AAG/D;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,CAAA;AAE/D;;GAEG;AACH,MAAM,MAAM,4CAA4C,GAAG;IACzD;;OAEG;IACH,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;IAC5C;;OAEG;IACH,GAAG,cAAc,EAAE;CACpB,CAAA;AAYD;;GAEG;AACH,qBAAa,mDAAmD,CAC9D,OAAO,SAAS,yDAAyD,GAAG,yDAAyD,CACrI,SAAQ,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,GAAG,cAAc,CAAC;IACvE,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA4F;IAC5I,gBAAyB,mBAAmB,EAAE,MAAM,CAAkE;IACtH,OAAgB,MAAM,EAAE,MAAM,CAG7B;IAED,IAAI,mBAAmB,WAEtB;IAED;;OAEG;IACH,SAAS,KAAK,eAAe,IAAI,MAAM,EAAE,CAGxC;cAEwB,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,CAAC,WAAW,EAAE,GAAG,cAAc,EAAE,CAAC,CAAC;IA4B7G;;;OAGG;cACa,oBAAoB,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAa7E;;;OAGG;cACa,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAmB9B,yBAAyB,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,EAAE,GAAG,SAAS,CAAC;CAOjI"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/StateToIndexCandidateDiviner/Diviner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAA;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAEnE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4CAA4C,CAAA;AAErF,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAA;AACxF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAA;AAC/E,OAAO,KAAK,EAAE,yDAAyD,EAAE,MAAM,8CAA8C,CAAA;AAE7H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EACV,MAAM,EAAoB,WAAW,EACtC,MAAM,2BAA2B,CAAA;AAGlC,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EACf,eAAe,EAChB,MAAM,4BAA4B,CAAA;AAGnC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAA;AAG/D;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,CAAA;AAE/D;;GAEG;AACH,MAAM,MAAM,4CAA4C,GAAG;IACzD;;OAEG;IACH,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;IAC5C;;OAEG;IACH,GAAG,cAAc,EAAE;CACpB,CAAA;AAYD;;GAEG;AACH,qBAAa,mDAAmD,CAC9D,OAAO,SAAS,yDAAyD,GAAG,yDAAyD,CACrI,SAAQ,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,GAAG,cAAc,CAAC;IACvE,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA4F;IAC5I,gBAAyB,mBAAmB,EAAE,MAAM,CAAkE;IACtH,OAAgB,MAAM,EAAE,MAAM,CAG7B;IAED,IAAI,mBAAmB,WAEtB;IAED;;OAEG;IACH,SAAS,KAAK,eAAe,IAAI,MAAM,EAAE,CAGxC;cAEwB,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,CAAC,WAAW,EAAE,GAAG,cAAc,EAAE,CAAC,CAAC;IA4B7G;;;OAGG;cACa,oBAAoB,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAa7E;;;OAGG;cACa,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAmB9B,yBAAyB,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,EAAE,GAAG,SAAS,CAAC;CAOjI"}

package/dist/browser/index.mjs

@@ -259,6 +259,7 @@ import { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from
 import { DivinerWrapper } from "@xyo-network/diviner-wrapper";
 import { isModuleState, ModuleStateSchema } from "@xyo-network/module-model";
 import { PayloadBuilder as PayloadBuilder3 } from "@xyo-network/payload-builder";
+import { StorageMetaConstants } from "@xyo-network/payload-model";
 import { intraBoundwitnessSchemaCombinations as intraBoundwitnessSchemaCombinations2 } from "@xyo-network/payload-utils";
 import { TimestampSchema } from "@xyo-network/witness-timestamp";
 var order = "asc";
@@ -295,20 +296,20 @@ var TemporalIndexingDivinerStateToIndexCandidateDiviner = class extends Abstract
       {
         schema: ModuleStateSchema,
         state: {
-          offset: 0
+          cursor: StorageMetaConstants.minLocalSequence
         }
       }
     ];
-    const { offset } = lastState.state;
+    const { cursor } = lastState.state;
     const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore();
     if (!boundWitnessDiviner) return [
       lastState
     ];
-    const query = await new PayloadBuilder3({
+    const query = new PayloadBuilder3({
       schema: BoundWitnessDivinerQuerySchema
     }).fields({
       limit: this.payloadDivinerLimit,
-      offset,
+      cursor,
       order,
       payload_schemas: this.payload_schemas
     }).build();
@@ -328,7 +329,7 @@ var TemporalIndexingDivinerStateToIndexCandidateDiviner = class extends Abstract
       schema: ModuleStateSchema,
       state: {
         ...lastState.state,
-        offset: offset + batch.length
+        cursor: batch.at(-1)?._sequence
       }
     };
     return [

package/dist/browser/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner.ts","../../src/DivinerQueryToIndexQueryDiviner/Diviner.ts","../../src/IndexCandidateToIndexDiviner/Diviner.ts","../../src/IndexQueryResponseToDivinerQueryResponseDiviner/Diviner.ts","../../src/StateToIndexCandidateDiviner/Diviner.ts"],"sourcesContent":["import { IndexingDiviner } from '@xyo-network/diviner-indexing-memory'\nimport type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport type { TemporalIndexingDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\nexport class TemporalIndexingDiviner<\n  TParams extends TemporalIndexingDivinerParams = TemporalIndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends IndexingDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerConfigSchema\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    return true\n  }\n}\n","import type { Hash } from '@xylabs/hex'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary, reducePayloads } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport type { TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { isPayloadOfSchemaType } from '@xyo-network/payload-model'\n// TODO: Inherit from JsonPathAggregateDiviner\n/**\n * A diviner that converts diviner query to index query\n */\nexport class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<\n  TParams extends TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas = [...super.configSchemas, TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema]\n  static override readonly defaultConfigSchema = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'divinerQueryToIndexQueryDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * The schema of the diviner query payloads\n   */\n  protected get divinerQuerySchema(): string {\n    return this.config.divinerQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index query payloads\n   */\n  protected get indexQuerySchema(): string {\n    return this.config.indexQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index payloads\n   */\n  protected get indexSchema(): string {\n    return this.config.indexSchema ?? TemporalIndexingDivinerResultIndexSchema\n  }\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return (\n      this.config?.schemaTransforms ?? {\n        [this.divinerQuerySchema]: [\n          {\n            defaultValue: 1,\n            destinationField: 'limit',\n            sourcePathExpression: '$.limit',\n          },\n          {\n            defaultValue: 0,\n            destinationField: 'offset',\n            sourcePathExpression: '$.offset',\n          },\n          {\n            defaultValue: 'desc',\n            destinationField: 'order',\n            sourcePathExpression: '$.order',\n          },\n        ],\n      }\n    )\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    const queries = payloads.filter(isPayloadOfSchemaType<PayloadDivinerQueryPayload>(this.divinerQuerySchema))\n    if (queries.length > 0) {\n      return await Promise.all(\n        queries.map(async (query) => {\n          const fields = await reducePayloads<PayloadDivinerQueryPayload & { sources?: Hash[] }>(\n            [query],\n            this.payloadTransformers,\n            this.indexQuerySchema,\n          )\n          // TODO: Make index schema configurable\n          fields.schemas = [this.indexSchema]\n          // TODO: Make sources not need to be deleted\n          delete fields.sources\n          // TODO: Add support for additional filters\n          return await new PayloadBuilder<Payload>({ schema: this.indexQuerySchema }).fields(fields).build()\n        }),\n      )\n    }\n    return []\n  }\n}\n","import { containsAll } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n  TemporalIndexingDivinerResultIndex,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\nimport { isAnyPayload } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\n\ntype IndexableHashes = [Hash, ...Hash[]]\n\nconst moduleName = 'TemporalIndexingDivinerIndexCandidateToIndexDiviner'\n\n/**\n * Diviner which transforms index candidates to indexes using JSON Path to map\n * source properties in the supplied payloads to destination fields in the\n * resultant index\n */\nexport class TemporalIndexingDivinerIndexCandidateToIndexDiviner<\n  TParams extends TemporalIndexingDivinerIndexCandidateToIndexDivinerParams = TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexCandidateToIndexDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return assertEx(this.config?.schemaTransforms, () => `${moduleName}: Missing config.schemaTransforms section`)\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // If the Bound Witness does not contain all the required schemas do not index it\n    const indexableBoundWitnesses = payloads\n      .filter(isBoundWitness)\n      .filter(bw => containsAll(bw.payload_schemas, this.indexableSchemas))\n    // If the Payload is not one of the indexable schemas do not index it\n    const indexablePayloads = payloads.filter(p => this.isIndexablePayload(p))\n    // If there is nothing to index, return an empty array\n    if (indexableBoundWitnesses.length === 0 || indexablePayloads.length === 0) return []\n    // Hash all the indexable data once\n    const [bwDictionary, payloadDictionary] = await Promise.all([\n      PayloadBuilder.toDataHashMap(indexableBoundWitnesses),\n      PayloadBuilder.toDataHashMap(indexablePayloads),\n    ])\n\n    // Initialize the array for validIndexableTuples outside of the loop\n    const validIndexableTuples: IndexableHashes[] = []\n\n    // Iterate over each entry in bwDictionary\n    for (const [bwHash, bw] of Object.entries(bwDictionary) as [Hash, BoundWitness][]) {\n      // Find the combinations of payloads that satisfy the required schemas\n      const combinations = intraBoundwitnessSchemaCombinations(bw, this.indexableSchemas)\n\n      // Iterate over each combination\n      for (const combination of combinations) {\n        const indexablePayloads = combination.map(hash => payloadDictionary[hash]).filter(exists)\n\n        // If we found the right amount of indexable payloads (of the correct schema as checked\n        // above) in this BW, then index it\n        if (indexablePayloads.length === this.indexableSchemas.length) {\n          validIndexableTuples.push([bwHash, ...combination])\n        }\n      }\n    }\n\n    // Create the indexes from the tuples\n    const indexes = await Promise.all(\n      validIndexableTuples.map<Promise<TemporalIndexingDivinerResultIndex>>(async ([bwHash, ...sourcePayloadHashes]) => {\n        const sourcePayloads = sourcePayloadHashes.map(hash => payloadDictionary[hash])\n        // Use the payload transformers to convert the fields from the source payloads to the destination fields\n        const indexFields = sourcePayloads.flatMap((payload) => {\n          // Find the transformers for this payload\n          const transformers = this.payloadTransformers[payload.schema]\n          // If transformers exist, apply them to the payload otherwise return an empty array\n          return transformers ? transformers.map(transform => transform(payload)) : []\n        })\n        // Include all the sources for reference\n        const sources: string[] = [bwHash, ...sourcePayloadHashes]\n        // Build and return the index\n        return await new PayloadBuilder<TemporalIndexingDivinerResultIndex>({ schema: TemporalIndexingDivinerResultIndexSchema })\n          .fields(Object.assign({ sources }, ...indexFields))\n          .build()\n      }),\n    )\n    return indexes.flat()\n  }\n\n  /**\n   * Identifies if a payload is one that is indexed by this diviner\n   * @param x The candidate payload\n   * @returns True if the payload is one indexed by this diviner, false otherwise\n   */\n  protected isIndexablePayload = (x: unknown) => {\n    return isAnyPayload(x) && this.indexableSchemas.includes(x?.schema)\n  }\n\n  /**\n   * Identifies if a schema is one that is indexed by this diviner\n   * @param schema The candidate schema\n   * @returns True if this schema is one indexed by this diviner, false otherwise\n   */\n  protected isIndexableSchema = (schema?: string | null) => {\n    return typeof schema === 'string' ? this.indexableSchemas.includes(schema) : false\n  }\n}\n","import { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\n/**\n * Transforms an ImageThumbnailIndex response into an ImageThumbnailResponse response\n */\nexport class TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner extends AbstractDiviner {\n  static override readonly configSchemas: Schema[] = [\n    ...super.configSchemas,\n    TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema,\n  ]\n\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexQueryResponseToDivinerQueryResponseDiviner',\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // NOTE: We're not doing anything with the query payloads but some diviners\n    // might want to use this to transform from the query to the response (for example\n    // if we use a plaintext value in the query to generate a hash key in the index)\n    // const queries = payloads.filter(isPayloadDivinerQueryPayload)\n    const responses = payloads.filter(p => !isPayloadDivinerQueryPayload(p))\n    return await Promise.resolve(responses)\n  }\n}\n","import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { ArchivistInstance } from '@xyo-network/archivist-model'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport type { BoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-abstract'\nimport type { BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'\nimport type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport type {\n  Labels, ModuleIdentifier, ModuleState,\n} from '@xyo-network/module-model'\nimport { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\nimport type { TimeStamp } from '@xyo-network/witness-timestamp'\nimport { TimestampSchema } from '@xyo-network/witness-timestamp'\n\n/**\n * All Payload types involved in index candidates for indexing\n */\nexport type IndexCandidate = BoundWitness | Payload | TimeStamp\n\n/**\n * The response from the TemporalStateToIndexCandidateDiviner\n */\nexport type TemporalStateToIndexCandidateDivinerResponse = [\n  /**\n   * The next state of the diviner\n   */\n  nextState: ModuleState<IndexingDivinerState>,\n  /**\n   * The index candidates\n   */\n  ...IndexCandidate[],\n]\n\n/**\n * The default order to search Bound Witnesses to identify index candidates\n */\nconst order = 'asc'\n\n/**\n * The name of the module (for logging purposes)\n */\nconst moduleName = 'TemporalIndexingDivinerStateToIndexCandidateDiviner'\n\n/**\n * Transforms candidates for image thumbnail indexing into their indexed representation\n */\nexport class TemporalIndexingDivinerStateToIndexCandidateDiviner<\n  TParams extends TemporalIndexingDivinerStateToIndexCandidateDivinerParams = TemporalIndexingDivinerStateToIndexCandidateDivinerParams,\n> extends AbstractDiviner<TParams, Payload, ModuleState | IndexCandidate> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'stateToIndexCandidateDiviner',\n  }\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1000\n  }\n\n  /**\n   * The required payload_schemas within BoundWitnesses to identify index candidates\n   */\n  protected get payload_schemas(): string[] {\n    const schemas = this.config.filter?.payload_schemas\n    return [TimestampSchema, ...(schemas ?? [])]\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<[ModuleState, ...IndexCandidate[]]> {\n    // Retrieve the last state from what was passed in\n    const lastState = payloads.find(isModuleState<IndexingDivinerState>)\n    // If there is no last state, start from the beginning\n    if (!lastState) return [{ schema: ModuleStateSchema, state: { offset: 0 } }]\n    // Otherwise, get the last offset\n    const { offset } = lastState.state\n    // Get next batch of results starting from the offset\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore()\n    if (!boundWitnessDiviner) return [lastState]\n    const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit, offset, order, payload_schemas: this.payload_schemas,\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return [lastState]\n    // Get source data\n    const sourceArchivist = await this.getArchivistForStore()\n    if (!sourceArchivist) return [lastState]\n    const bws = batch.filter(isBoundWitness)\n    const indexCandidates: IndexCandidate[] = (await Promise.all(bws.map(bw => this.getPayloadsInBoundWitness(bw, sourceArchivist))))\n      .filter(exists)\n      .flat()\n    const nextState = { schema: ModuleStateSchema, state: { ...lastState.state, offset: offset + batch.length } }\n    return [nextState, ...indexCandidates]\n  }\n\n  /**\n   * Retrieves the archivist for the payloadStore\n   * @returns The archivist for the payloadStore or undefined if not resolvable\n   */\n  protected async getArchivistForStore(): Promise<ArchivistWrapper | undefined> {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.archivist,\n      () => `${moduleName}: Config for payloadStore.archivist not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return undefined\n    // Return the wrapped archivist\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the payloadStore\n   * @returns The BoundWitness Diviner for the payloadStore or undefined if not resolvable\n   */\n  protected async getBoundWitnessDivinerForStore() {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.boundWitnessDiviner,\n      () => `${moduleName}: Config for payloadStore.boundWitnessDiviner not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return\n    // Return the wrapped diviner\n    return DivinerWrapper.wrap<\n      DivinerWrapper<\n        BoundWitnessDiviner<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, BoundWitness>,\n        BoundWitnessDivinerQueryPayload,\n        BoundWitness\n      >\n    >(mod, this.account)\n  }\n\n  protected async getPayloadsInBoundWitness(bw: BoundWitness, archivist: ArchivistInstance): Promise<IndexCandidate[] | undefined> {\n    const combinations = intraBoundwitnessSchemaCombinations(bw, this.payload_schemas).flat()\n    if (combinations.length === 0) return undefined\n    const hashes = new Set(combinations)\n    const indexCandidates = await archivist.get([...hashes])\n    return [bw, ...indexCandidates]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,uBAAuB;AAGhC,SAASC,2CAA2C;AAG7C,IAAMC,0BAAN,cASGC,gBAAAA;EAfV,OAeUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAEvD,MAAyBE,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,WAAO;EACT;AACF;;;ACtBA,SAASC,uBAAuB;AAChC,SAASC,kCAAkCC,sBAAsB;AAGjE,SAASC,iCAAiC;AAE1C,SACEC,oEACAC,gDACK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,6BAA6B;AAK/B,IAAMC,yDAAN,cAEGC,gBAAAA;EApBV,OAoBUA;;;EACR,OAAyBC,gBAAgB;OAAI,MAAMA;IAAeC;;EAClE,OAAyBC,sBAAsBD;EAC/C,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,qBAA6B;AACzC,WAAO,KAAKC,OAAOD,sBAAsBE;EAC3C;;;;EAKA,IAAcC,mBAA2B;AACvC,WAAO,KAAKF,OAAOE,oBAAoBD;EACzC;;;;EAKA,IAAcE,cAAsB;AAClC,WAAO,KAAKH,OAAOG,eAAeC;EACpC;;;;EAKA,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKR,kBAAmB,MAAKA,oBAAoBS,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKX;EACd;;;;;EAMA,IAAcY,sBAA6D;AACzE,QAAI,CAAC,KAAKX,qBAAsB,MAAKA,uBAAuBY,iCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKV;EACd;;;;;EAMA,IAAcU,mBAAmE;AAC/E,WACE,KAAKR,QAAQQ,oBAAoB;MAC/B,CAAC,KAAKT,kBAAkB,GAAG;QACzB;UACEY,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;;IAEJ;EAEJ;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AACnF,UAAMC,UAAUD,SAASE,OAAOC,sBAAkD,KAAKnB,kBAAkB,CAAA;AACzG,QAAIiB,QAAQG,SAAS,GAAG;AACtB,aAAO,MAAMC,QAAQC,IACnBL,QAAQM,IAAI,OAAOC,UAAAA;AACjB,cAAMC,SAAS,MAAMC,eACnB;UAACF;WACD,KAAKd,qBACL,KAAKP,gBAAgB;AAGvBsB,eAAOE,UAAU;UAAC,KAAKvB;;AAEvB,eAAOqB,OAAOG;AAEd,eAAO,MAAM,IAAIC,eAAwB;UAAEC,QAAQ,KAAK3B;QAAiB,CAAA,EAAGsB,OAAOA,MAAAA,EAAQM,MAAK;MAClG,CAAA,CAAA;IAEJ;AACA,WAAO,CAAA;EACT;AACF;;;ACvHA,SAASC,mBAAmB;AAC5B,SAASC,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,sBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAChC,SAASC,oCAAAA,yCAAwC;AAMjD,SACEC,iEACAC,4CAAAA,iDACK;AAEP,SAASC,kBAAAA,uBAAsB;AAE/B,SAASC,oBAAoB;AAC7B,SAASC,2CAA2C;AAIpD,IAAMC,aAAa;AAOZ,IAAMC,sDAAN,cAEGC,iBAAAA;EAlCV,OAkCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKF,kBAAmB,MAAKA,oBAAoBG,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKL;EACd;;;;;EAMA,IAAcM,sBAA6D;AACzE,QAAI,CAAC,KAAKL,qBAAsB,MAAKA,uBAAuBM,kCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKJ;EACd;;;;;EAMA,IAAcI,mBAAmE;AAC/E,WAAOG,SAAS,KAAKC,QAAQJ,kBAAkB,MAAM,GAAGZ,UAAAA,2CAAqD;EAC/G;EAEA,MAAyBiB,cAAcC,WAAsB,CAAA,GAAwB;AAEnF,UAAMC,0BAA0BD,SAC7BE,OAAOC,cAAAA,EACPD,OAAOE,CAAAA,OAAMC,YAAYD,GAAGE,iBAAiB,KAAKf,gBAAgB,CAAA;AAErE,UAAMgB,oBAAoBP,SAASE,OAAOM,CAAAA,MAAK,KAAKC,mBAAmBD,CAAAA,CAAAA;AAEvE,QAAIP,wBAAwBS,WAAW,KAAKH,kBAAkBG,WAAW,EAAG,QAAO,CAAA;AAEnF,UAAM,CAACC,cAAcC,iBAAAA,IAAqB,MAAMC,QAAQC,IAAI;MAC1DC,gBAAeC,cAAcf,uBAAAA;MAC7Bc,gBAAeC,cAAcT,iBAAAA;KAC9B;AAGD,UAAMU,uBAA0C,CAAA;AAGhD,eAAW,CAACC,QAAQd,EAAAA,KAAOZ,OAAO2B,QAAQR,YAAAA,GAAyC;AAEjF,YAAMS,eAAeC,oCAAoCjB,IAAI,KAAKb,gBAAgB;AAGlF,iBAAW+B,eAAeF,cAAc;AACtC,cAAMb,qBAAoBe,YAAYC,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK,EAAEtB,OAAOuB,MAAAA;AAIlF,YAAIlB,mBAAkBG,WAAW,KAAKnB,iBAAiBmB,QAAQ;AAC7DO,+BAAqBS,KAAK;YAACR;eAAWI;WAAY;QACpD;MACF;IACF;AAGA,UAAMK,UAAU,MAAMd,QAAQC,IAC5BG,qBAAqBM,IAAiD,OAAO,CAACL,QAAWU,sBAAAA,MAAoB;AAC3G,YAAMC,iBAAiBD,oBAAoBL,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK;AAE9E,YAAMM,cAAcD,eAAeE,QAAQ,CAACC,YAAAA;AAE1C,cAAMC,eAAe,KAAKtC,oBAAoBqC,QAAQE,MAAM;AAE5D,eAAOD,eAAeA,aAAaV,IAAIY,CAAAA,cAAaA,UAAUH,OAAAA,CAAAA,IAAY,CAAA;MAC5E,CAAA;AAEA,YAAMI,UAAoB;QAAClB;WAAWU;;AAEtC,aAAO,MAAM,IAAIb,gBAAmD;QAAEmB,QAAQG;MAAyC,CAAA,EACpHC,OAAO9C,OAAO+C,OAAO;QAAEH;MAAQ,GAAA,GAAMN,WAAAA,CAAAA,EACrCU,MAAK;IACV,CAAA,CAAA;AAEF,WAAOb,QAAQc,KAAI;EACrB;;;;;;EAOUhC,qBAAqB,wBAACiC,MAAAA;AAC9B,WAAOC,aAAaD,CAAAA,KAAM,KAAKnD,iBAAiBqD,SAASF,GAAGR,MAAAA;EAC9D,GAF+B;;;;;;EASrBW,oBAAoB,wBAACX,WAAAA;AAC7B,WAAO,OAAOA,WAAW,WAAW,KAAK3C,iBAAiBqD,SAASV,MAAAA,IAAU;EAC/E,GAF8B;AAGhC;;;AChJA,SAASY,mBAAAA,wBAAuB;AAChC,SAASC,oCAAoC;AAC7C,SAASC,0FAA0F;AAO5F,IAAMC,yEAAN,cAAqFC,iBAAAA;EAT5F,OAS4FA;;;EAC1F,OAAyBC,gBAA0B;OAC9C,MAAMA;IACTC;;EAGF,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AAKnF,UAAMC,YAAYD,SAASE,OAAOC,CAAAA,MAAK,CAACC,6BAA6BD,CAAAA,CAAAA;AACrE,WAAO,MAAME,QAAQC,QAAQL,SAAAA;EAC/B;AACF;;;AC7BA,SAASM,YAAAA,iBAAgB;AACzB,SAASC,UAAAA,eAAc;AAEvB,SAASC,wBAAwB;AAEjC,SAASC,kBAAAA,uBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAGhC,SAASC,sCAAsC;AAG/C,SAASC,uEAAuE;AAChF,SAASC,sBAAsB;AAI/B,SAASC,eAAeC,yBAAyB;AACjD,SAASC,kBAAAA,uBAAsB;AAE/B,SAASC,uCAAAA,4CAA2C;AAEpD,SAASC,uBAAuB;AAwBhC,IAAMC,QAAQ;AAKd,IAAMC,cAAa;AAKZ,IAAMC,sDAAN,cAEGC,iBAAAA;EA1DV,OA0DUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;;;;EAKA,IAAcE,kBAA4B;AACxC,UAAMC,UAAU,KAAKF,OAAOG,QAAQF;AACpC,WAAO;MAACG;SAAqBF,WAAW,CAAA;;EAC1C;EAEA,MAAyBG,cAAcC,WAAsB,CAAA,GAAiD;AAE5G,UAAMC,YAAYD,SAASE,KAAKC,aAAAA;AAEhC,QAAI,CAACF,UAAW,QAAO;MAAC;QAAEG,QAAQC;QAAmBC,OAAO;UAAEC,QAAQ;QAAE;MAAE;;AAE1E,UAAM,EAAEA,OAAM,IAAKN,UAAUK;AAE7B,UAAME,sBAAsB,MAAM,KAAKC,+BAA8B;AACrE,QAAI,CAACD,oBAAqB,QAAO;MAACP;;AAClC,UAAMS,QAAQ,MAAM,IAAIC,gBAAgD;MAAEP,QAAQQ;IAA+B,CAAA,EAC9GC,OAAO;MACNC,OAAO,KAAKrB;MAAqBc;MAAQtB;MAAOU,iBAAiB,KAAKA;IACxE,CAAA,EACCoB,MAAK;AACR,UAAMC,QAAQ,MAAMR,oBAAoBS,OAAO;MAACP;KAAM;AACtD,QAAIM,MAAME,WAAW,EAAG,QAAO;MAACjB;;AAEhC,UAAMkB,kBAAkB,MAAM,KAAKC,qBAAoB;AACvD,QAAI,CAACD,gBAAiB,QAAO;MAAClB;;AAC9B,UAAMoB,MAAML,MAAMnB,OAAOyB,eAAAA;AACzB,UAAMC,mBAAqC,MAAMC,QAAQC,IAAIJ,IAAIK,IAAIC,CAAAA,OAAM,KAAKC,0BAA0BD,IAAIR,eAAAA,CAAAA,CAAAA,GAC3GtB,OAAOgC,OAAAA,EACPC,KAAI;AACP,UAAMC,YAAY;MAAE3B,QAAQC;MAAmBC,OAAO;QAAE,GAAGL,UAAUK;QAAOC,QAAQA,SAASS,MAAME;MAAO;IAAE;AAC5G,WAAO;MAACa;SAAcR;;EACxB;;;;;EAMA,MAAgBH,uBAA8D;AAE5E,UAAMY,OAAyBC,UAC7B,KAAKvC,QAAQwC,cAAcC,WAC3B,MAAM,GAAGjD,WAAAA,mDAA6D;AAGxE,UAAMkD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK,QAAOE;AAEjB,WAAOC,iBAAiBC,KAAKJ,KAAK,KAAKK,OAAO;EAChD;;;;;EAMA,MAAgBhC,iCAAiC;AAE/C,UAAMuB,OAAyBC,UAC7B,KAAKvC,QAAQwC,cAAc1B,qBAC3B,MAAM,GAAGtB,WAAAA,6DAAuE;AAGlF,UAAMkD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK;AAEV,WAAOM,eAAeF,KAMpBJ,KAAK,KAAKK,OAAO;EACrB;EAEA,MAAgBb,0BAA0BD,IAAkBQ,WAAqE;AAC/H,UAAMQ,eAAeC,qCAAoCjB,IAAI,KAAKhC,eAAe,EAAEmC,KAAI;AACvF,QAAIa,aAAazB,WAAW,EAAG,QAAOoB;AACtC,UAAMO,SAAS,IAAIC,IAAIH,YAAAA;AACvB,UAAMpB,kBAAkB,MAAMY,UAAUY,IAAI;SAAIF;KAAO;AACvD,WAAO;MAAClB;SAAOJ;;EACjB;AACF;","names":["IndexingDiviner","TemporalIndexingDivinerConfigSchema","TemporalIndexingDiviner","IndexingDiviner","configSchemas","TemporalIndexingDivinerConfigSchema","defaultConfigSchema","startHandler","AbstractDiviner","jsonPathToTransformersDictionary","reducePayloads","PayloadDivinerQuerySchema","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isPayloadOfSchemaType","TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","divinerQuerySchema","config","PayloadDivinerQuerySchema","indexQuerySchema","indexSchema","TemporalIndexingDivinerResultIndexSchema","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","defaultValue","destinationField","sourcePathExpression","divineHandler","payloads","queries","filter","isPayloadOfSchemaType","length","Promise","all","map","query","fields","reducePayloads","schemas","sources","PayloadBuilder","schema","build","containsAll","assertEx","exists","isBoundWitness","AbstractDiviner","jsonPathToTransformersDictionary","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isAnyPayload","intraBoundwitnessSchemaCombinations","moduleName","TemporalIndexingDivinerIndexCandidateToIndexDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","assertEx","config","divineHandler","payloads","indexableBoundWitnesses","filter","isBoundWitness","bw","containsAll","payload_schemas","indexablePayloads","p","isIndexablePayload","length","bwDictionary","payloadDictionary","Promise","all","PayloadBuilder","toDataHashMap","validIndexableTuples","bwHash","entries","combinations","intraBoundwitnessSchemaCombinations","combination","map","hash","exists","push","indexes","sourcePayloadHashes","sourcePayloads","indexFields","flatMap","payload","transformers","schema","transform","sources","TemporalIndexingDivinerResultIndexSchema","fields","assign","build","flat","x","isAnyPayload","includes","isIndexableSchema","AbstractDiviner","isPayloadDivinerQueryPayload","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","defaultConfigSchema","labels","divineHandler","payloads","responses","filter","p","isPayloadDivinerQueryPayload","Promise","resolve","assertEx","exists","ArchivistWrapper","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","DivinerWrapper","isModuleState","ModuleStateSchema","PayloadBuilder","intraBoundwitnessSchemaCombinations","TimestampSchema","order","moduleName","TemporalIndexingDivinerStateToIndexCandidateDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","defaultConfigSchema","labels","payloadDivinerLimit","config","payload_schemas","schemas","filter","TimestampSchema","divineHandler","payloads","lastState","find","isModuleState","schema","ModuleStateSchema","state","offset","boundWitnessDiviner","getBoundWitnessDivinerForStore","query","PayloadBuilder","BoundWitnessDivinerQuerySchema","fields","limit","build","batch","divine","length","sourceArchivist","getArchivistForStore","bws","isBoundWitness","indexCandidates","Promise","all","map","bw","getPayloadsInBoundWitness","exists","flat","nextState","name","assertEx","payloadStore","archivist","mod","resolve","undefined","ArchivistWrapper","wrap","account","DivinerWrapper","combinations","intraBoundwitnessSchemaCombinations","hashes","Set","get"]}
+{"version":3,"sources":["../../src/Diviner.ts","../../src/DivinerQueryToIndexQueryDiviner/Diviner.ts","../../src/IndexCandidateToIndexDiviner/Diviner.ts","../../src/IndexQueryResponseToDivinerQueryResponseDiviner/Diviner.ts","../../src/StateToIndexCandidateDiviner/Diviner.ts"],"sourcesContent":["import { IndexingDiviner } from '@xyo-network/diviner-indexing-memory'\nimport type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport type { TemporalIndexingDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\nexport class TemporalIndexingDiviner<\n  TParams extends TemporalIndexingDivinerParams = TemporalIndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends IndexingDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerConfigSchema\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    return true\n  }\n}\n","import type { Hash } from '@xylabs/hex'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary, reducePayloads } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport type { TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { isPayloadOfSchemaType } from '@xyo-network/payload-model'\n// TODO: Inherit from JsonPathAggregateDiviner\n/**\n * A diviner that converts diviner query to index query\n */\nexport class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<\n  TParams extends TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas = [...super.configSchemas, TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema]\n  static override readonly defaultConfigSchema = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'divinerQueryToIndexQueryDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * The schema of the diviner query payloads\n   */\n  protected get divinerQuerySchema(): string {\n    return this.config.divinerQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index query payloads\n   */\n  protected get indexQuerySchema(): string {\n    return this.config.indexQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index payloads\n   */\n  protected get indexSchema(): string {\n    return this.config.indexSchema ?? TemporalIndexingDivinerResultIndexSchema\n  }\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return (\n      this.config?.schemaTransforms ?? {\n        [this.divinerQuerySchema]: [\n          {\n            defaultValue: 1,\n            destinationField: 'limit',\n            sourcePathExpression: '$.limit',\n          },\n          {\n            defaultValue: 0,\n            destinationField: 'offset',\n            sourcePathExpression: '$.offset',\n          },\n          {\n            defaultValue: 'desc',\n            destinationField: 'order',\n            sourcePathExpression: '$.order',\n          },\n        ],\n      }\n    )\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    const queries = payloads.filter(isPayloadOfSchemaType<PayloadDivinerQueryPayload>(this.divinerQuerySchema))\n    if (queries.length > 0) {\n      return await Promise.all(\n        queries.map(async (query) => {\n          const fields = await reducePayloads<PayloadDivinerQueryPayload & { sources?: Hash[] }>(\n            [query],\n            this.payloadTransformers,\n            this.indexQuerySchema,\n          )\n          // TODO: Make index schema configurable\n          fields.schemas = [this.indexSchema]\n          // TODO: Make sources not need to be deleted\n          delete fields.sources\n          // TODO: Add support for additional filters\n          return await new PayloadBuilder<Payload>({ schema: this.indexQuerySchema }).fields(fields).build()\n        }),\n      )\n    }\n    return []\n  }\n}\n","import { containsAll } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n  TemporalIndexingDivinerResultIndex,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\nimport { isAnyPayload } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\n\ntype IndexableHashes = [Hash, ...Hash[]]\n\nconst moduleName = 'TemporalIndexingDivinerIndexCandidateToIndexDiviner'\n\n/**\n * Diviner which transforms index candidates to indexes using JSON Path to map\n * source properties in the supplied payloads to destination fields in the\n * resultant index\n */\nexport class TemporalIndexingDivinerIndexCandidateToIndexDiviner<\n  TParams extends TemporalIndexingDivinerIndexCandidateToIndexDivinerParams = TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexCandidateToIndexDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return assertEx(this.config?.schemaTransforms, () => `${moduleName}: Missing config.schemaTransforms section`)\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // If the Bound Witness does not contain all the required schemas do not index it\n    const indexableBoundWitnesses = payloads\n      .filter(isBoundWitness)\n      .filter(bw => containsAll(bw.payload_schemas, this.indexableSchemas))\n    // If the Payload is not one of the indexable schemas do not index it\n    const indexablePayloads = payloads.filter(p => this.isIndexablePayload(p))\n    // If there is nothing to index, return an empty array\n    if (indexableBoundWitnesses.length === 0 || indexablePayloads.length === 0) return []\n    // Hash all the indexable data once\n    const [bwDictionary, payloadDictionary] = await Promise.all([\n      PayloadBuilder.toDataHashMap(indexableBoundWitnesses),\n      PayloadBuilder.toDataHashMap(indexablePayloads),\n    ])\n\n    // Initialize the array for validIndexableTuples outside of the loop\n    const validIndexableTuples: IndexableHashes[] = []\n\n    // Iterate over each entry in bwDictionary\n    for (const [bwHash, bw] of Object.entries(bwDictionary) as [Hash, BoundWitness][]) {\n      // Find the combinations of payloads that satisfy the required schemas\n      const combinations = intraBoundwitnessSchemaCombinations(bw, this.indexableSchemas)\n\n      // Iterate over each combination\n      for (const combination of combinations) {\n        const indexablePayloads = combination.map(hash => payloadDictionary[hash]).filter(exists)\n\n        // If we found the right amount of indexable payloads (of the correct schema as checked\n        // above) in this BW, then index it\n        if (indexablePayloads.length === this.indexableSchemas.length) {\n          validIndexableTuples.push([bwHash, ...combination])\n        }\n      }\n    }\n\n    // Create the indexes from the tuples\n    const indexes = await Promise.all(\n      validIndexableTuples.map<Promise<TemporalIndexingDivinerResultIndex>>(async ([bwHash, ...sourcePayloadHashes]) => {\n        const sourcePayloads = sourcePayloadHashes.map(hash => payloadDictionary[hash])\n        // Use the payload transformers to convert the fields from the source payloads to the destination fields\n        const indexFields = sourcePayloads.flatMap((payload) => {\n          // Find the transformers for this payload\n          const transformers = this.payloadTransformers[payload.schema]\n          // If transformers exist, apply them to the payload otherwise return an empty array\n          return transformers ? transformers.map(transform => transform(payload)) : []\n        })\n        // Include all the sources for reference\n        const sources: string[] = [bwHash, ...sourcePayloadHashes]\n        // Build and return the index\n        return await new PayloadBuilder<TemporalIndexingDivinerResultIndex>({ schema: TemporalIndexingDivinerResultIndexSchema })\n          .fields(Object.assign({ sources }, ...indexFields))\n          .build()\n      }),\n    )\n    return indexes.flat()\n  }\n\n  /**\n   * Identifies if a payload is one that is indexed by this diviner\n   * @param x The candidate payload\n   * @returns True if the payload is one indexed by this diviner, false otherwise\n   */\n  protected isIndexablePayload = (x: unknown) => {\n    return isAnyPayload(x) && this.indexableSchemas.includes(x?.schema)\n  }\n\n  /**\n   * Identifies if a schema is one that is indexed by this diviner\n   * @param schema The candidate schema\n   * @returns True if this schema is one indexed by this diviner, false otherwise\n   */\n  protected isIndexableSchema = (schema?: string | null) => {\n    return typeof schema === 'string' ? this.indexableSchemas.includes(schema) : false\n  }\n}\n","import { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\n/**\n * Transforms an ImageThumbnailIndex response into an ImageThumbnailResponse response\n */\nexport class TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner extends AbstractDiviner {\n  static override readonly configSchemas: Schema[] = [\n    ...super.configSchemas,\n    TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema,\n  ]\n\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexQueryResponseToDivinerQueryResponseDiviner',\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // NOTE: We're not doing anything with the query payloads but some diviners\n    // might want to use this to transform from the query to the response (for example\n    // if we use a plaintext value in the query to generate a hash key in the index)\n    // const queries = payloads.filter(isPayloadDivinerQueryPayload)\n    const responses = payloads.filter(p => !isPayloadDivinerQueryPayload(p))\n    return await Promise.resolve(responses)\n  }\n}\n","import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hex } from '@xylabs/hex'\nimport type { ArchivistInstance } from '@xyo-network/archivist-model'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport type { BoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-abstract'\nimport type { BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'\nimport type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport type {\n  Labels, ModuleIdentifier, ModuleState,\n} from '@xyo-network/module-model'\nimport { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema,\n  WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport { StorageMetaConstants } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\nimport type { TimeStamp } from '@xyo-network/witness-timestamp'\nimport { TimestampSchema } from '@xyo-network/witness-timestamp'\n\n/**\n * All Payload types involved in index candidates for indexing\n */\nexport type IndexCandidate = BoundWitness | Payload | TimeStamp\n\n/**\n * The response from the TemporalStateToIndexCandidateDiviner\n */\nexport type TemporalStateToIndexCandidateDivinerResponse = [\n  /**\n   * The next state of the diviner\n   */\n  nextState: ModuleState<IndexingDivinerState>,\n  /**\n   * The index candidates\n   */\n  ...IndexCandidate[],\n]\n\n/**\n * The default order to search Bound Witnesses to identify index candidates\n */\nconst order = 'asc'\n\n/**\n * The name of the module (for logging purposes)\n */\nconst moduleName = 'TemporalIndexingDivinerStateToIndexCandidateDiviner'\n\n/**\n * Transforms candidates for image thumbnail indexing into their indexed representation\n */\nexport class TemporalIndexingDivinerStateToIndexCandidateDiviner<\n  TParams extends TemporalIndexingDivinerStateToIndexCandidateDivinerParams = TemporalIndexingDivinerStateToIndexCandidateDivinerParams,\n> extends AbstractDiviner<TParams, Payload, ModuleState | IndexCandidate> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'stateToIndexCandidateDiviner',\n  }\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1000\n  }\n\n  /**\n   * The required payload_schemas within BoundWitnesses to identify index candidates\n   */\n  protected get payload_schemas(): string[] {\n    const schemas = this.config.filter?.payload_schemas\n    return [TimestampSchema, ...(schemas ?? [])]\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<[ModuleState, ...IndexCandidate[]]> {\n    // Retrieve the last state from what was passed in\n    const lastState = payloads.find(isModuleState<IndexingDivinerState>)\n    // If there is no last state, start from the beginning\n    if (!lastState) return [{ schema: ModuleStateSchema, state: { cursor: StorageMetaConstants.minLocalSequence } }]\n    // Otherwise, get the last offset\n    const { cursor } = lastState.state\n    // Get next batch of results starting from the offset\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore()\n    if (!boundWitnessDiviner) return [lastState]\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit, cursor: cursor as Hex, order, payload_schemas: this.payload_schemas,\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return [lastState]\n    // Get source data\n    const sourceArchivist = await this.getArchivistForStore()\n    if (!sourceArchivist) return [lastState]\n    const bws = batch.filter(isBoundWitness)\n    const indexCandidates: IndexCandidate[] = (await Promise.all(bws.map(bw => this.getPayloadsInBoundWitness(bw, sourceArchivist))))\n      .filter(exists)\n      .flat()\n    const nextState = { schema: ModuleStateSchema, state: { ...lastState.state, cursor: batch.at(-1)?._sequence } }\n    return [nextState, ...indexCandidates]\n  }\n\n  /**\n   * Retrieves the archivist for the payloadStore\n   * @returns The archivist for the payloadStore or undefined if not resolvable\n   */\n  protected async getArchivistForStore(): Promise<ArchivistWrapper | undefined> {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.archivist,\n      () => `${moduleName}: Config for payloadStore.archivist not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return undefined\n    // Return the wrapped archivist\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the payloadStore\n   * @returns The BoundWitness Diviner for the payloadStore or undefined if not resolvable\n   */\n  protected async getBoundWitnessDivinerForStore() {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.boundWitnessDiviner,\n      () => `${moduleName}: Config for payloadStore.boundWitnessDiviner not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return\n    // Return the wrapped diviner\n    return DivinerWrapper.wrap<\n      DivinerWrapper<\n        BoundWitnessDiviner<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, BoundWitness>,\n        BoundWitnessDivinerQueryPayload,\n        WithStorageMeta<BoundWitness>\n      >\n    >(mod, this.account)\n  }\n\n  protected async getPayloadsInBoundWitness(bw: BoundWitness, archivist: ArchivistInstance): Promise<IndexCandidate[] | undefined> {\n    const combinations = intraBoundwitnessSchemaCombinations(bw, this.payload_schemas).flat()\n    if (combinations.length === 0) return undefined\n    const hashes = new Set(combinations)\n    const indexCandidates = await archivist.get([...hashes])\n    return [bw, ...indexCandidates]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,uBAAuB;AAGhC,SAASC,2CAA2C;AAG7C,IAAMC,0BAAN,cASGC,gBAAAA;EAfV,OAeUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAEvD,MAAyBE,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,WAAO;EACT;AACF;;;ACtBA,SAASC,uBAAuB;AAChC,SAASC,kCAAkCC,sBAAsB;AAGjE,SAASC,iCAAiC;AAE1C,SACEC,oEACAC,gDACK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,6BAA6B;AAK/B,IAAMC,yDAAN,cAEGC,gBAAAA;EApBV,OAoBUA;;;EACR,OAAyBC,gBAAgB;OAAI,MAAMA;IAAeC;;EAClE,OAAyBC,sBAAsBD;EAC/C,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,qBAA6B;AACzC,WAAO,KAAKC,OAAOD,sBAAsBE;EAC3C;;;;EAKA,IAAcC,mBAA2B;AACvC,WAAO,KAAKF,OAAOE,oBAAoBD;EACzC;;;;EAKA,IAAcE,cAAsB;AAClC,WAAO,KAAKH,OAAOG,eAAeC;EACpC;;;;EAKA,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKR,kBAAmB,MAAKA,oBAAoBS,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKX;EACd;;;;;EAMA,IAAcY,sBAA6D;AACzE,QAAI,CAAC,KAAKX,qBAAsB,MAAKA,uBAAuBY,iCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKV;EACd;;;;;EAMA,IAAcU,mBAAmE;AAC/E,WACE,KAAKR,QAAQQ,oBAAoB;MAC/B,CAAC,KAAKT,kBAAkB,GAAG;QACzB;UACEY,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;;IAEJ;EAEJ;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AACnF,UAAMC,UAAUD,SAASE,OAAOC,sBAAkD,KAAKnB,kBAAkB,CAAA;AACzG,QAAIiB,QAAQG,SAAS,GAAG;AACtB,aAAO,MAAMC,QAAQC,IACnBL,QAAQM,IAAI,OAAOC,UAAAA;AACjB,cAAMC,SAAS,MAAMC,eACnB;UAACF;WACD,KAAKd,qBACL,KAAKP,gBAAgB;AAGvBsB,eAAOE,UAAU;UAAC,KAAKvB;;AAEvB,eAAOqB,OAAOG;AAEd,eAAO,MAAM,IAAIC,eAAwB;UAAEC,QAAQ,KAAK3B;QAAiB,CAAA,EAAGsB,OAAOA,MAAAA,EAAQM,MAAK;MAClG,CAAA,CAAA;IAEJ;AACA,WAAO,CAAA;EACT;AACF;;;ACvHA,SAASC,mBAAmB;AAC5B,SAASC,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,sBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAChC,SAASC,oCAAAA,yCAAwC;AAMjD,SACEC,iEACAC,4CAAAA,iDACK;AAEP,SAASC,kBAAAA,uBAAsB;AAE/B,SAASC,oBAAoB;AAC7B,SAASC,2CAA2C;AAIpD,IAAMC,aAAa;AAOZ,IAAMC,sDAAN,cAEGC,iBAAAA;EAlCV,OAkCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKF,kBAAmB,MAAKA,oBAAoBG,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKL;EACd;;;;;EAMA,IAAcM,sBAA6D;AACzE,QAAI,CAAC,KAAKL,qBAAsB,MAAKA,uBAAuBM,kCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKJ;EACd;;;;;EAMA,IAAcI,mBAAmE;AAC/E,WAAOG,SAAS,KAAKC,QAAQJ,kBAAkB,MAAM,GAAGZ,UAAAA,2CAAqD;EAC/G;EAEA,MAAyBiB,cAAcC,WAAsB,CAAA,GAAwB;AAEnF,UAAMC,0BAA0BD,SAC7BE,OAAOC,cAAAA,EACPD,OAAOE,CAAAA,OAAMC,YAAYD,GAAGE,iBAAiB,KAAKf,gBAAgB,CAAA;AAErE,UAAMgB,oBAAoBP,SAASE,OAAOM,CAAAA,MAAK,KAAKC,mBAAmBD,CAAAA,CAAAA;AAEvE,QAAIP,wBAAwBS,WAAW,KAAKH,kBAAkBG,WAAW,EAAG,QAAO,CAAA;AAEnF,UAAM,CAACC,cAAcC,iBAAAA,IAAqB,MAAMC,QAAQC,IAAI;MAC1DC,gBAAeC,cAAcf,uBAAAA;MAC7Bc,gBAAeC,cAAcT,iBAAAA;KAC9B;AAGD,UAAMU,uBAA0C,CAAA;AAGhD,eAAW,CAACC,QAAQd,EAAAA,KAAOZ,OAAO2B,QAAQR,YAAAA,GAAyC;AAEjF,YAAMS,eAAeC,oCAAoCjB,IAAI,KAAKb,gBAAgB;AAGlF,iBAAW+B,eAAeF,cAAc;AACtC,cAAMb,qBAAoBe,YAAYC,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK,EAAEtB,OAAOuB,MAAAA;AAIlF,YAAIlB,mBAAkBG,WAAW,KAAKnB,iBAAiBmB,QAAQ;AAC7DO,+BAAqBS,KAAK;YAACR;eAAWI;WAAY;QACpD;MACF;IACF;AAGA,UAAMK,UAAU,MAAMd,QAAQC,IAC5BG,qBAAqBM,IAAiD,OAAO,CAACL,QAAWU,sBAAAA,MAAoB;AAC3G,YAAMC,iBAAiBD,oBAAoBL,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK;AAE9E,YAAMM,cAAcD,eAAeE,QAAQ,CAACC,YAAAA;AAE1C,cAAMC,eAAe,KAAKtC,oBAAoBqC,QAAQE,MAAM;AAE5D,eAAOD,eAAeA,aAAaV,IAAIY,CAAAA,cAAaA,UAAUH,OAAAA,CAAAA,IAAY,CAAA;MAC5E,CAAA;AAEA,YAAMI,UAAoB;QAAClB;WAAWU;;AAEtC,aAAO,MAAM,IAAIb,gBAAmD;QAAEmB,QAAQG;MAAyC,CAAA,EACpHC,OAAO9C,OAAO+C,OAAO;QAAEH;MAAQ,GAAA,GAAMN,WAAAA,CAAAA,EACrCU,MAAK;IACV,CAAA,CAAA;AAEF,WAAOb,QAAQc,KAAI;EACrB;;;;;;EAOUhC,qBAAqB,wBAACiC,MAAAA;AAC9B,WAAOC,aAAaD,CAAAA,KAAM,KAAKnD,iBAAiBqD,SAASF,GAAGR,MAAAA;EAC9D,GAF+B;;;;;;EASrBW,oBAAoB,wBAACX,WAAAA;AAC7B,WAAO,OAAOA,WAAW,WAAW,KAAK3C,iBAAiBqD,SAASV,MAAAA,IAAU;EAC/E,GAF8B;AAGhC;;;AChJA,SAASY,mBAAAA,wBAAuB;AAChC,SAASC,oCAAoC;AAC7C,SAASC,0FAA0F;AAO5F,IAAMC,yEAAN,cAAqFC,iBAAAA;EAT5F,OAS4FA;;;EAC1F,OAAyBC,gBAA0B;OAC9C,MAAMA;IACTC;;EAGF,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AAKnF,UAAMC,YAAYD,SAASE,OAAOC,CAAAA,MAAK,CAACC,6BAA6BD,CAAAA,CAAAA;AACrE,WAAO,MAAME,QAAQC,QAAQL,SAAAA;EAC/B;AACF;;;AC7BA,SAASM,YAAAA,iBAAgB;AACzB,SAASC,UAAAA,eAAc;AAGvB,SAASC,wBAAwB;AAEjC,SAASC,kBAAAA,uBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAGhC,SAASC,sCAAsC;AAG/C,SAASC,uEAAuE;AAChF,SAASC,sBAAsB;AAI/B,SAASC,eAAeC,yBAAyB;AACjD,SAASC,kBAAAA,uBAAsB;AAK/B,SAASC,4BAA4B;AACrC,SAASC,uCAAAA,4CAA2C;AAEpD,SAASC,uBAAuB;AAwBhC,IAAMC,QAAQ;AAKd,IAAMC,cAAa;AAKZ,IAAMC,sDAAN,cAEGC,iBAAAA;EA/DV,OA+DUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;;;;EAKA,IAAcE,kBAA4B;AACxC,UAAMC,UAAU,KAAKF,OAAOG,QAAQF;AACpC,WAAO;MAACG;SAAqBF,WAAW,CAAA;;EAC1C;EAEA,MAAyBG,cAAcC,WAAsB,CAAA,GAAiD;AAE5G,UAAMC,YAAYD,SAASE,KAAKC,aAAAA;AAEhC,QAAI,CAACF,UAAW,QAAO;MAAC;QAAEG,QAAQC;QAAmBC,OAAO;UAAEC,QAAQC,qBAAqBC;QAAiB;MAAE;;AAE9G,UAAM,EAAEF,OAAM,IAAKN,UAAUK;AAE7B,UAAMI,sBAAsB,MAAM,KAAKC,+BAA8B;AACrE,QAAI,CAACD,oBAAqB,QAAO;MAACT;;AAClC,UAAMW,QAAQ,IAAIC,gBAAgD;MAAET,QAAQU;IAA+B,CAAA,EACxGC,OAAO;MACNC,OAAO,KAAKvB;MAAqBc;MAAuBtB;MAAOU,iBAAiB,KAAKA;IACvF,CAAA,EACCsB,MAAK;AACR,UAAMC,QAAQ,MAAMR,oBAAoBS,OAAO;MAACP;KAAM;AACtD,QAAIM,MAAME,WAAW,EAAG,QAAO;MAACnB;;AAEhC,UAAMoB,kBAAkB,MAAM,KAAKC,qBAAoB;AACvD,QAAI,CAACD,gBAAiB,QAAO;MAACpB;;AAC9B,UAAMsB,MAAML,MAAMrB,OAAO2B,eAAAA;AACzB,UAAMC,mBAAqC,MAAMC,QAAQC,IAAIJ,IAAIK,IAAIC,CAAAA,OAAM,KAAKC,0BAA0BD,IAAIR,eAAAA,CAAAA,CAAAA,GAC3GxB,OAAOkC,OAAAA,EACPC,KAAI;AACP,UAAMC,YAAY;MAAE7B,QAAQC;MAAmBC,OAAO;QAAE,GAAGL,UAAUK;QAAOC,QAAQW,MAAMgB,GAAG,EAAC,GAAIC;MAAU;IAAE;AAC9G,WAAO;MAACF;SAAcR;;EACxB;;;;;EAMA,MAAgBH,uBAA8D;AAE5E,UAAMc,OAAyBC,UAC7B,KAAK3C,QAAQ4C,cAAcC,WAC3B,MAAM,GAAGrD,WAAAA,mDAA6D;AAGxE,UAAMsD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK,QAAOE;AAEjB,WAAOC,iBAAiBC,KAAKJ,KAAK,KAAKK,OAAO;EAChD;;;;;EAMA,MAAgBlC,iCAAiC;AAE/C,UAAMyB,OAAyBC,UAC7B,KAAK3C,QAAQ4C,cAAc5B,qBAC3B,MAAM,GAAGxB,WAAAA,6DAAuE;AAGlF,UAAMsD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK;AAEV,WAAOM,eAAeF,KAMpBJ,KAAK,KAAKK,OAAO;EACrB;EAEA,MAAgBf,0BAA0BD,IAAkBU,WAAqE;AAC/H,UAAMQ,eAAeC,qCAAoCnB,IAAI,KAAKlC,eAAe,EAAEqC,KAAI;AACvF,QAAIe,aAAa3B,WAAW,EAAG,QAAOsB;AACtC,UAAMO,SAAS,IAAIC,IAAIH,YAAAA;AACvB,UAAMtB,kBAAkB,MAAMc,UAAUY,IAAI;SAAIF;KAAO;AACvD,WAAO;MAACpB;SAAOJ;;EACjB;AACF;","names":["IndexingDiviner","TemporalIndexingDivinerConfigSchema","TemporalIndexingDiviner","IndexingDiviner","configSchemas","TemporalIndexingDivinerConfigSchema","defaultConfigSchema","startHandler","AbstractDiviner","jsonPathToTransformersDictionary","reducePayloads","PayloadDivinerQuerySchema","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isPayloadOfSchemaType","TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","divinerQuerySchema","config","PayloadDivinerQuerySchema","indexQuerySchema","indexSchema","TemporalIndexingDivinerResultIndexSchema","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","defaultValue","destinationField","sourcePathExpression","divineHandler","payloads","queries","filter","isPayloadOfSchemaType","length","Promise","all","map","query","fields","reducePayloads","schemas","sources","PayloadBuilder","schema","build","containsAll","assertEx","exists","isBoundWitness","AbstractDiviner","jsonPathToTransformersDictionary","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isAnyPayload","intraBoundwitnessSchemaCombinations","moduleName","TemporalIndexingDivinerIndexCandidateToIndexDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","assertEx","config","divineHandler","payloads","indexableBoundWitnesses","filter","isBoundWitness","bw","containsAll","payload_schemas","indexablePayloads","p","isIndexablePayload","length","bwDictionary","payloadDictionary","Promise","all","PayloadBuilder","toDataHashMap","validIndexableTuples","bwHash","entries","combinations","intraBoundwitnessSchemaCombinations","combination","map","hash","exists","push","indexes","sourcePayloadHashes","sourcePayloads","indexFields","flatMap","payload","transformers","schema","transform","sources","TemporalIndexingDivinerResultIndexSchema","fields","assign","build","flat","x","isAnyPayload","includes","isIndexableSchema","AbstractDiviner","isPayloadDivinerQueryPayload","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","defaultConfigSchema","labels","divineHandler","payloads","responses","filter","p","isPayloadDivinerQueryPayload","Promise","resolve","assertEx","exists","ArchivistWrapper","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","DivinerWrapper","isModuleState","ModuleStateSchema","PayloadBuilder","StorageMetaConstants","intraBoundwitnessSchemaCombinations","TimestampSchema","order","moduleName","TemporalIndexingDivinerStateToIndexCandidateDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","defaultConfigSchema","labels","payloadDivinerLimit","config","payload_schemas","schemas","filter","TimestampSchema","divineHandler","payloads","lastState","find","isModuleState","schema","ModuleStateSchema","state","cursor","StorageMetaConstants","minLocalSequence","boundWitnessDiviner","getBoundWitnessDivinerForStore","query","PayloadBuilder","BoundWitnessDivinerQuerySchema","fields","limit","build","batch","divine","length","sourceArchivist","getArchivistForStore","bws","isBoundWitness","indexCandidates","Promise","all","map","bw","getPayloadsInBoundWitness","exists","flat","nextState","at","_sequence","name","assertEx","payloadStore","archivist","mod","resolve","undefined","ArchivistWrapper","wrap","account","DivinerWrapper","combinations","intraBoundwitnessSchemaCombinations","hashes","Set","get"]}

package/dist/neutral/StateToIndexCandidateDiviner/Diviner.d.ts

@@ -8,7 +8,7 @@ import type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model';
 import type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model';
 import { DivinerWrapper } from '@xyo-network/diviner-wrapper';
 import type { Labels, ModuleState } from '@xyo-network/module-model';
-import type { Payload, Schema } from '@xyo-network/payload-model';
+import type { Payload, Schema, WithStorageMeta } from '@xyo-network/payload-model';
 import type { TimeStamp } from '@xyo-network/witness-timestamp';
 /**
  * All Payload types involved in index candidates for indexing
@@ -101,7 +101,7 @@ export declare class TemporalIndexingDivinerStateToIndexCandidateDiviner<TParams
         schema: BoundWitnessDivinerQuerySchema;
     } & import("@xyo-network/diviner-boundwitness-model").BoundWitnessDivinerPredicate & import("@xyo-network/payload-model").QueryFields, "schema"> & {
         schema: "network.xyo.diviner.boundwitness.query";
-    }, BoundWitness> | undefined>;
+    }, WithStorageMeta<BoundWitness>> | undefined>;
     protected getPayloadsInBoundWitness(bw: BoundWitness, archivist: ArchivistInstance): Promise<IndexCandidate[] | undefined>;
 }
 //# sourceMappingURL=Diviner.d.ts.map

package/dist/neutral/StateToIndexCandidateDiviner/Diviner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/StateToIndexCandidateDiviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAA;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAEnE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4CAA4C,CAAA;AAErF,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAA;AACxF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAA;AAC/E,OAAO,KAAK,EAAE,yDAAyD,EAAE,MAAM,8CAA8C,CAAA;AAE7H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EACV,MAAM,EAAoB,WAAW,EACtC,MAAM,2BAA2B,CAAA;AAGlC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAA;AAEjE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAA;AAG/D;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,CAAA;AAE/D;;GAEG;AACH,MAAM,MAAM,4CAA4C,GAAG;IACzD;;OAEG;IACH,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;IAC5C;;OAEG;IACH,GAAG,cAAc,EAAE;CACpB,CAAA;AAYD;;GAEG;AACH,qBAAa,mDAAmD,CAC9D,OAAO,SAAS,yDAAyD,GAAG,yDAAyD,CACrI,SAAQ,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,GAAG,cAAc,CAAC;IACvE,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA4F;IAC5I,gBAAyB,mBAAmB,EAAE,MAAM,CAAkE;IACtH,OAAgB,MAAM,EAAE,MAAM,CAG7B;IAED,IAAI,mBAAmB,WAEtB;IAED;;OAEG;IACH,SAAS,KAAK,eAAe,IAAI,MAAM,EAAE,CAGxC;cAEwB,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,CAAC,WAAW,EAAE,GAAG,cAAc,EAAE,CAAC,CAAC;IA4B7G;;;OAGG;cACa,oBAAoB,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAa7E;;;OAGG;cACa,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAmB9B,yBAAyB,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,EAAE,GAAG,SAAS,CAAC;CAOjI"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/StateToIndexCandidateDiviner/Diviner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAA;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAEnE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4CAA4C,CAAA;AAErF,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAA;AACxF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAA;AAC/E,OAAO,KAAK,EAAE,yDAAyD,EAAE,MAAM,8CAA8C,CAAA;AAE7H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EACV,MAAM,EAAoB,WAAW,EACtC,MAAM,2BAA2B,CAAA;AAGlC,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EACf,eAAe,EAChB,MAAM,4BAA4B,CAAA;AAGnC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAA;AAG/D;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,CAAA;AAE/D;;GAEG;AACH,MAAM,MAAM,4CAA4C,GAAG;IACzD;;OAEG;IACH,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;IAC5C;;OAEG;IACH,GAAG,cAAc,EAAE;CACpB,CAAA;AAYD;;GAEG;AACH,qBAAa,mDAAmD,CAC9D,OAAO,SAAS,yDAAyD,GAAG,yDAAyD,CACrI,SAAQ,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,GAAG,cAAc,CAAC;IACvE,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA4F;IAC5I,gBAAyB,mBAAmB,EAAE,MAAM,CAAkE;IACtH,OAAgB,MAAM,EAAE,MAAM,CAG7B;IAED,IAAI,mBAAmB,WAEtB;IAED;;OAEG;IACH,SAAS,KAAK,eAAe,IAAI,MAAM,EAAE,CAGxC;cAEwB,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,CAAC,WAAW,EAAE,GAAG,cAAc,EAAE,CAAC,CAAC;IA4B7G;;;OAGG;cACa,oBAAoB,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAa7E;;;OAGG;cACa,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAmB9B,yBAAyB,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,EAAE,GAAG,SAAS,CAAC;CAOjI"}

package/dist/neutral/index.mjs

@@ -259,6 +259,7 @@ import { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from
 import { DivinerWrapper } from "@xyo-network/diviner-wrapper";
 import { isModuleState, ModuleStateSchema } from "@xyo-network/module-model";
 import { PayloadBuilder as PayloadBuilder3 } from "@xyo-network/payload-builder";
+import { StorageMetaConstants } from "@xyo-network/payload-model";
 import { intraBoundwitnessSchemaCombinations as intraBoundwitnessSchemaCombinations2 } from "@xyo-network/payload-utils";
 import { TimestampSchema } from "@xyo-network/witness-timestamp";
 var order = "asc";
@@ -295,20 +296,20 @@ var TemporalIndexingDivinerStateToIndexCandidateDiviner = class extends Abstract
       {
         schema: ModuleStateSchema,
         state: {
-          offset: 0
+          cursor: StorageMetaConstants.minLocalSequence
         }
       }
     ];
-    const { offset } = lastState.state;
+    const { cursor } = lastState.state;
     const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore();
     if (!boundWitnessDiviner) return [
       lastState
     ];
-    const query = await new PayloadBuilder3({
+    const query = new PayloadBuilder3({
       schema: BoundWitnessDivinerQuerySchema
     }).fields({
       limit: this.payloadDivinerLimit,
-      offset,
+      cursor,
       order,
       payload_schemas: this.payload_schemas
     }).build();
@@ -328,7 +329,7 @@ var TemporalIndexingDivinerStateToIndexCandidateDiviner = class extends Abstract
       schema: ModuleStateSchema,
       state: {
         ...lastState.state,
-        offset: offset + batch.length
+        cursor: batch.at(-1)?._sequence
       }
     };
     return [

package/dist/neutral/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner.ts","../../src/DivinerQueryToIndexQueryDiviner/Diviner.ts","../../src/IndexCandidateToIndexDiviner/Diviner.ts","../../src/IndexQueryResponseToDivinerQueryResponseDiviner/Diviner.ts","../../src/StateToIndexCandidateDiviner/Diviner.ts"],"sourcesContent":["import { IndexingDiviner } from '@xyo-network/diviner-indexing-memory'\nimport type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport type { TemporalIndexingDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\nexport class TemporalIndexingDiviner<\n  TParams extends TemporalIndexingDivinerParams = TemporalIndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends IndexingDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerConfigSchema\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    return true\n  }\n}\n","import type { Hash } from '@xylabs/hex'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary, reducePayloads } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport type { TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { isPayloadOfSchemaType } from '@xyo-network/payload-model'\n// TODO: Inherit from JsonPathAggregateDiviner\n/**\n * A diviner that converts diviner query to index query\n */\nexport class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<\n  TParams extends TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas = [...super.configSchemas, TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema]\n  static override readonly defaultConfigSchema = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'divinerQueryToIndexQueryDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * The schema of the diviner query payloads\n   */\n  protected get divinerQuerySchema(): string {\n    return this.config.divinerQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index query payloads\n   */\n  protected get indexQuerySchema(): string {\n    return this.config.indexQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index payloads\n   */\n  protected get indexSchema(): string {\n    return this.config.indexSchema ?? TemporalIndexingDivinerResultIndexSchema\n  }\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return (\n      this.config?.schemaTransforms ?? {\n        [this.divinerQuerySchema]: [\n          {\n            defaultValue: 1,\n            destinationField: 'limit',\n            sourcePathExpression: '$.limit',\n          },\n          {\n            defaultValue: 0,\n            destinationField: 'offset',\n            sourcePathExpression: '$.offset',\n          },\n          {\n            defaultValue: 'desc',\n            destinationField: 'order',\n            sourcePathExpression: '$.order',\n          },\n        ],\n      }\n    )\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    const queries = payloads.filter(isPayloadOfSchemaType<PayloadDivinerQueryPayload>(this.divinerQuerySchema))\n    if (queries.length > 0) {\n      return await Promise.all(\n        queries.map(async (query) => {\n          const fields = await reducePayloads<PayloadDivinerQueryPayload & { sources?: Hash[] }>(\n            [query],\n            this.payloadTransformers,\n            this.indexQuerySchema,\n          )\n          // TODO: Make index schema configurable\n          fields.schemas = [this.indexSchema]\n          // TODO: Make sources not need to be deleted\n          delete fields.sources\n          // TODO: Add support for additional filters\n          return await new PayloadBuilder<Payload>({ schema: this.indexQuerySchema }).fields(fields).build()\n        }),\n      )\n    }\n    return []\n  }\n}\n","import { containsAll } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n  TemporalIndexingDivinerResultIndex,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\nimport { isAnyPayload } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\n\ntype IndexableHashes = [Hash, ...Hash[]]\n\nconst moduleName = 'TemporalIndexingDivinerIndexCandidateToIndexDiviner'\n\n/**\n * Diviner which transforms index candidates to indexes using JSON Path to map\n * source properties in the supplied payloads to destination fields in the\n * resultant index\n */\nexport class TemporalIndexingDivinerIndexCandidateToIndexDiviner<\n  TParams extends TemporalIndexingDivinerIndexCandidateToIndexDivinerParams = TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexCandidateToIndexDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return assertEx(this.config?.schemaTransforms, () => `${moduleName}: Missing config.schemaTransforms section`)\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // If the Bound Witness does not contain all the required schemas do not index it\n    const indexableBoundWitnesses = payloads\n      .filter(isBoundWitness)\n      .filter(bw => containsAll(bw.payload_schemas, this.indexableSchemas))\n    // If the Payload is not one of the indexable schemas do not index it\n    const indexablePayloads = payloads.filter(p => this.isIndexablePayload(p))\n    // If there is nothing to index, return an empty array\n    if (indexableBoundWitnesses.length === 0 || indexablePayloads.length === 0) return []\n    // Hash all the indexable data once\n    const [bwDictionary, payloadDictionary] = await Promise.all([\n      PayloadBuilder.toDataHashMap(indexableBoundWitnesses),\n      PayloadBuilder.toDataHashMap(indexablePayloads),\n    ])\n\n    // Initialize the array for validIndexableTuples outside of the loop\n    const validIndexableTuples: IndexableHashes[] = []\n\n    // Iterate over each entry in bwDictionary\n    for (const [bwHash, bw] of Object.entries(bwDictionary) as [Hash, BoundWitness][]) {\n      // Find the combinations of payloads that satisfy the required schemas\n      const combinations = intraBoundwitnessSchemaCombinations(bw, this.indexableSchemas)\n\n      // Iterate over each combination\n      for (const combination of combinations) {\n        const indexablePayloads = combination.map(hash => payloadDictionary[hash]).filter(exists)\n\n        // If we found the right amount of indexable payloads (of the correct schema as checked\n        // above) in this BW, then index it\n        if (indexablePayloads.length === this.indexableSchemas.length) {\n          validIndexableTuples.push([bwHash, ...combination])\n        }\n      }\n    }\n\n    // Create the indexes from the tuples\n    const indexes = await Promise.all(\n      validIndexableTuples.map<Promise<TemporalIndexingDivinerResultIndex>>(async ([bwHash, ...sourcePayloadHashes]) => {\n        const sourcePayloads = sourcePayloadHashes.map(hash => payloadDictionary[hash])\n        // Use the payload transformers to convert the fields from the source payloads to the destination fields\n        const indexFields = sourcePayloads.flatMap((payload) => {\n          // Find the transformers for this payload\n          const transformers = this.payloadTransformers[payload.schema]\n          // If transformers exist, apply them to the payload otherwise return an empty array\n          return transformers ? transformers.map(transform => transform(payload)) : []\n        })\n        // Include all the sources for reference\n        const sources: string[] = [bwHash, ...sourcePayloadHashes]\n        // Build and return the index\n        return await new PayloadBuilder<TemporalIndexingDivinerResultIndex>({ schema: TemporalIndexingDivinerResultIndexSchema })\n          .fields(Object.assign({ sources }, ...indexFields))\n          .build()\n      }),\n    )\n    return indexes.flat()\n  }\n\n  /**\n   * Identifies if a payload is one that is indexed by this diviner\n   * @param x The candidate payload\n   * @returns True if the payload is one indexed by this diviner, false otherwise\n   */\n  protected isIndexablePayload = (x: unknown) => {\n    return isAnyPayload(x) && this.indexableSchemas.includes(x?.schema)\n  }\n\n  /**\n   * Identifies if a schema is one that is indexed by this diviner\n   * @param schema The candidate schema\n   * @returns True if this schema is one indexed by this diviner, false otherwise\n   */\n  protected isIndexableSchema = (schema?: string | null) => {\n    return typeof schema === 'string' ? this.indexableSchemas.includes(schema) : false\n  }\n}\n","import { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\n/**\n * Transforms an ImageThumbnailIndex response into an ImageThumbnailResponse response\n */\nexport class TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner extends AbstractDiviner {\n  static override readonly configSchemas: Schema[] = [\n    ...super.configSchemas,\n    TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema,\n  ]\n\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexQueryResponseToDivinerQueryResponseDiviner',\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // NOTE: We're not doing anything with the query payloads but some diviners\n    // might want to use this to transform from the query to the response (for example\n    // if we use a plaintext value in the query to generate a hash key in the index)\n    // const queries = payloads.filter(isPayloadDivinerQueryPayload)\n    const responses = payloads.filter(p => !isPayloadDivinerQueryPayload(p))\n    return await Promise.resolve(responses)\n  }\n}\n","import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { ArchivistInstance } from '@xyo-network/archivist-model'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport type { BoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-abstract'\nimport type { BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'\nimport type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport type {\n  Labels, ModuleIdentifier, ModuleState,\n} from '@xyo-network/module-model'\nimport { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\nimport type { TimeStamp } from '@xyo-network/witness-timestamp'\nimport { TimestampSchema } from '@xyo-network/witness-timestamp'\n\n/**\n * All Payload types involved in index candidates for indexing\n */\nexport type IndexCandidate = BoundWitness | Payload | TimeStamp\n\n/**\n * The response from the TemporalStateToIndexCandidateDiviner\n */\nexport type TemporalStateToIndexCandidateDivinerResponse = [\n  /**\n   * The next state of the diviner\n   */\n  nextState: ModuleState<IndexingDivinerState>,\n  /**\n   * The index candidates\n   */\n  ...IndexCandidate[],\n]\n\n/**\n * The default order to search Bound Witnesses to identify index candidates\n */\nconst order = 'asc'\n\n/**\n * The name of the module (for logging purposes)\n */\nconst moduleName = 'TemporalIndexingDivinerStateToIndexCandidateDiviner'\n\n/**\n * Transforms candidates for image thumbnail indexing into their indexed representation\n */\nexport class TemporalIndexingDivinerStateToIndexCandidateDiviner<\n  TParams extends TemporalIndexingDivinerStateToIndexCandidateDivinerParams = TemporalIndexingDivinerStateToIndexCandidateDivinerParams,\n> extends AbstractDiviner<TParams, Payload, ModuleState | IndexCandidate> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'stateToIndexCandidateDiviner',\n  }\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1000\n  }\n\n  /**\n   * The required payload_schemas within BoundWitnesses to identify index candidates\n   */\n  protected get payload_schemas(): string[] {\n    const schemas = this.config.filter?.payload_schemas\n    return [TimestampSchema, ...(schemas ?? [])]\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<[ModuleState, ...IndexCandidate[]]> {\n    // Retrieve the last state from what was passed in\n    const lastState = payloads.find(isModuleState<IndexingDivinerState>)\n    // If there is no last state, start from the beginning\n    if (!lastState) return [{ schema: ModuleStateSchema, state: { offset: 0 } }]\n    // Otherwise, get the last offset\n    const { offset } = lastState.state\n    // Get next batch of results starting from the offset\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore()\n    if (!boundWitnessDiviner) return [lastState]\n    const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit, offset, order, payload_schemas: this.payload_schemas,\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return [lastState]\n    // Get source data\n    const sourceArchivist = await this.getArchivistForStore()\n    if (!sourceArchivist) return [lastState]\n    const bws = batch.filter(isBoundWitness)\n    const indexCandidates: IndexCandidate[] = (await Promise.all(bws.map(bw => this.getPayloadsInBoundWitness(bw, sourceArchivist))))\n      .filter(exists)\n      .flat()\n    const nextState = { schema: ModuleStateSchema, state: { ...lastState.state, offset: offset + batch.length } }\n    return [nextState, ...indexCandidates]\n  }\n\n  /**\n   * Retrieves the archivist for the payloadStore\n   * @returns The archivist for the payloadStore or undefined if not resolvable\n   */\n  protected async getArchivistForStore(): Promise<ArchivistWrapper | undefined> {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.archivist,\n      () => `${moduleName}: Config for payloadStore.archivist not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return undefined\n    // Return the wrapped archivist\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the payloadStore\n   * @returns The BoundWitness Diviner for the payloadStore or undefined if not resolvable\n   */\n  protected async getBoundWitnessDivinerForStore() {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.boundWitnessDiviner,\n      () => `${moduleName}: Config for payloadStore.boundWitnessDiviner not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return\n    // Return the wrapped diviner\n    return DivinerWrapper.wrap<\n      DivinerWrapper<\n        BoundWitnessDiviner<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, BoundWitness>,\n        BoundWitnessDivinerQueryPayload,\n        BoundWitness\n      >\n    >(mod, this.account)\n  }\n\n  protected async getPayloadsInBoundWitness(bw: BoundWitness, archivist: ArchivistInstance): Promise<IndexCandidate[] | undefined> {\n    const combinations = intraBoundwitnessSchemaCombinations(bw, this.payload_schemas).flat()\n    if (combinations.length === 0) return undefined\n    const hashes = new Set(combinations)\n    const indexCandidates = await archivist.get([...hashes])\n    return [bw, ...indexCandidates]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,uBAAuB;AAGhC,SAASC,2CAA2C;AAG7C,IAAMC,0BAAN,cASGC,gBAAAA;EAfV,OAeUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAEvD,MAAyBE,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,WAAO;EACT;AACF;;;ACtBA,SAASC,uBAAuB;AAChC,SAASC,kCAAkCC,sBAAsB;AAGjE,SAASC,iCAAiC;AAE1C,SACEC,oEACAC,gDACK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,6BAA6B;AAK/B,IAAMC,yDAAN,cAEGC,gBAAAA;EApBV,OAoBUA;;;EACR,OAAyBC,gBAAgB;OAAI,MAAMA;IAAeC;;EAClE,OAAyBC,sBAAsBD;EAC/C,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,qBAA6B;AACzC,WAAO,KAAKC,OAAOD,sBAAsBE;EAC3C;;;;EAKA,IAAcC,mBAA2B;AACvC,WAAO,KAAKF,OAAOE,oBAAoBD;EACzC;;;;EAKA,IAAcE,cAAsB;AAClC,WAAO,KAAKH,OAAOG,eAAeC;EACpC;;;;EAKA,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKR,kBAAmB,MAAKA,oBAAoBS,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKX;EACd;;;;;EAMA,IAAcY,sBAA6D;AACzE,QAAI,CAAC,KAAKX,qBAAsB,MAAKA,uBAAuBY,iCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKV;EACd;;;;;EAMA,IAAcU,mBAAmE;AAC/E,WACE,KAAKR,QAAQQ,oBAAoB;MAC/B,CAAC,KAAKT,kBAAkB,GAAG;QACzB;UACEY,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;;IAEJ;EAEJ;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AACnF,UAAMC,UAAUD,SAASE,OAAOC,sBAAkD,KAAKnB,kBAAkB,CAAA;AACzG,QAAIiB,QAAQG,SAAS,GAAG;AACtB,aAAO,MAAMC,QAAQC,IACnBL,QAAQM,IAAI,OAAOC,UAAAA;AACjB,cAAMC,SAAS,MAAMC,eACnB;UAACF;WACD,KAAKd,qBACL,KAAKP,gBAAgB;AAGvBsB,eAAOE,UAAU;UAAC,KAAKvB;;AAEvB,eAAOqB,OAAOG;AAEd,eAAO,MAAM,IAAIC,eAAwB;UAAEC,QAAQ,KAAK3B;QAAiB,CAAA,EAAGsB,OAAOA,MAAAA,EAAQM,MAAK;MAClG,CAAA,CAAA;IAEJ;AACA,WAAO,CAAA;EACT;AACF;;;ACvHA,SAASC,mBAAmB;AAC5B,SAASC,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,sBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAChC,SAASC,oCAAAA,yCAAwC;AAMjD,SACEC,iEACAC,4CAAAA,iDACK;AAEP,SAASC,kBAAAA,uBAAsB;AAE/B,SAASC,oBAAoB;AAC7B,SAASC,2CAA2C;AAIpD,IAAMC,aAAa;AAOZ,IAAMC,sDAAN,cAEGC,iBAAAA;EAlCV,OAkCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKF,kBAAmB,MAAKA,oBAAoBG,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKL;EACd;;;;;EAMA,IAAcM,sBAA6D;AACzE,QAAI,CAAC,KAAKL,qBAAsB,MAAKA,uBAAuBM,kCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKJ;EACd;;;;;EAMA,IAAcI,mBAAmE;AAC/E,WAAOG,SAAS,KAAKC,QAAQJ,kBAAkB,MAAM,GAAGZ,UAAAA,2CAAqD;EAC/G;EAEA,MAAyBiB,cAAcC,WAAsB,CAAA,GAAwB;AAEnF,UAAMC,0BAA0BD,SAC7BE,OAAOC,cAAAA,EACPD,OAAOE,CAAAA,OAAMC,YAAYD,GAAGE,iBAAiB,KAAKf,gBAAgB,CAAA;AAErE,UAAMgB,oBAAoBP,SAASE,OAAOM,CAAAA,MAAK,KAAKC,mBAAmBD,CAAAA,CAAAA;AAEvE,QAAIP,wBAAwBS,WAAW,KAAKH,kBAAkBG,WAAW,EAAG,QAAO,CAAA;AAEnF,UAAM,CAACC,cAAcC,iBAAAA,IAAqB,MAAMC,QAAQC,IAAI;MAC1DC,gBAAeC,cAAcf,uBAAAA;MAC7Bc,gBAAeC,cAAcT,iBAAAA;KAC9B;AAGD,UAAMU,uBAA0C,CAAA;AAGhD,eAAW,CAACC,QAAQd,EAAAA,KAAOZ,OAAO2B,QAAQR,YAAAA,GAAyC;AAEjF,YAAMS,eAAeC,oCAAoCjB,IAAI,KAAKb,gBAAgB;AAGlF,iBAAW+B,eAAeF,cAAc;AACtC,cAAMb,qBAAoBe,YAAYC,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK,EAAEtB,OAAOuB,MAAAA;AAIlF,YAAIlB,mBAAkBG,WAAW,KAAKnB,iBAAiBmB,QAAQ;AAC7DO,+BAAqBS,KAAK;YAACR;eAAWI;WAAY;QACpD;MACF;IACF;AAGA,UAAMK,UAAU,MAAMd,QAAQC,IAC5BG,qBAAqBM,IAAiD,OAAO,CAACL,QAAWU,sBAAAA,MAAoB;AAC3G,YAAMC,iBAAiBD,oBAAoBL,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK;AAE9E,YAAMM,cAAcD,eAAeE,QAAQ,CAACC,YAAAA;AAE1C,cAAMC,eAAe,KAAKtC,oBAAoBqC,QAAQE,MAAM;AAE5D,eAAOD,eAAeA,aAAaV,IAAIY,CAAAA,cAAaA,UAAUH,OAAAA,CAAAA,IAAY,CAAA;MAC5E,CAAA;AAEA,YAAMI,UAAoB;QAAClB;WAAWU;;AAEtC,aAAO,MAAM,IAAIb,gBAAmD;QAAEmB,QAAQG;MAAyC,CAAA,EACpHC,OAAO9C,OAAO+C,OAAO;QAAEH;MAAQ,GAAA,GAAMN,WAAAA,CAAAA,EACrCU,MAAK;IACV,CAAA,CAAA;AAEF,WAAOb,QAAQc,KAAI;EACrB;;;;;;EAOUhC,qBAAqB,wBAACiC,MAAAA;AAC9B,WAAOC,aAAaD,CAAAA,KAAM,KAAKnD,iBAAiBqD,SAASF,GAAGR,MAAAA;EAC9D,GAF+B;;;;;;EASrBW,oBAAoB,wBAACX,WAAAA;AAC7B,WAAO,OAAOA,WAAW,WAAW,KAAK3C,iBAAiBqD,SAASV,MAAAA,IAAU;EAC/E,GAF8B;AAGhC;;;AChJA,SAASY,mBAAAA,wBAAuB;AAChC,SAASC,oCAAoC;AAC7C,SAASC,0FAA0F;AAO5F,IAAMC,yEAAN,cAAqFC,iBAAAA;EAT5F,OAS4FA;;;EAC1F,OAAyBC,gBAA0B;OAC9C,MAAMA;IACTC;;EAGF,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AAKnF,UAAMC,YAAYD,SAASE,OAAOC,CAAAA,MAAK,CAACC,6BAA6BD,CAAAA,CAAAA;AACrE,WAAO,MAAME,QAAQC,QAAQL,SAAAA;EAC/B;AACF;;;AC7BA,SAASM,YAAAA,iBAAgB;AACzB,SAASC,UAAAA,eAAc;AAEvB,SAASC,wBAAwB;AAEjC,SAASC,kBAAAA,uBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAGhC,SAASC,sCAAsC;AAG/C,SAASC,uEAAuE;AAChF,SAASC,sBAAsB;AAI/B,SAASC,eAAeC,yBAAyB;AACjD,SAASC,kBAAAA,uBAAsB;AAE/B,SAASC,uCAAAA,4CAA2C;AAEpD,SAASC,uBAAuB;AAwBhC,IAAMC,QAAQ;AAKd,IAAMC,cAAa;AAKZ,IAAMC,sDAAN,cAEGC,iBAAAA;EA1DV,OA0DUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;;;;EAKA,IAAcE,kBAA4B;AACxC,UAAMC,UAAU,KAAKF,OAAOG,QAAQF;AACpC,WAAO;MAACG;SAAqBF,WAAW,CAAA;;EAC1C;EAEA,MAAyBG,cAAcC,WAAsB,CAAA,GAAiD;AAE5G,UAAMC,YAAYD,SAASE,KAAKC,aAAAA;AAEhC,QAAI,CAACF,UAAW,QAAO;MAAC;QAAEG,QAAQC;QAAmBC,OAAO;UAAEC,QAAQ;QAAE;MAAE;;AAE1E,UAAM,EAAEA,OAAM,IAAKN,UAAUK;AAE7B,UAAME,sBAAsB,MAAM,KAAKC,+BAA8B;AACrE,QAAI,CAACD,oBAAqB,QAAO;MAACP;;AAClC,UAAMS,QAAQ,MAAM,IAAIC,gBAAgD;MAAEP,QAAQQ;IAA+B,CAAA,EAC9GC,OAAO;MACNC,OAAO,KAAKrB;MAAqBc;MAAQtB;MAAOU,iBAAiB,KAAKA;IACxE,CAAA,EACCoB,MAAK;AACR,UAAMC,QAAQ,MAAMR,oBAAoBS,OAAO;MAACP;KAAM;AACtD,QAAIM,MAAME,WAAW,EAAG,QAAO;MAACjB;;AAEhC,UAAMkB,kBAAkB,MAAM,KAAKC,qBAAoB;AACvD,QAAI,CAACD,gBAAiB,QAAO;MAAClB;;AAC9B,UAAMoB,MAAML,MAAMnB,OAAOyB,eAAAA;AACzB,UAAMC,mBAAqC,MAAMC,QAAQC,IAAIJ,IAAIK,IAAIC,CAAAA,OAAM,KAAKC,0BAA0BD,IAAIR,eAAAA,CAAAA,CAAAA,GAC3GtB,OAAOgC,OAAAA,EACPC,KAAI;AACP,UAAMC,YAAY;MAAE3B,QAAQC;MAAmBC,OAAO;QAAE,GAAGL,UAAUK;QAAOC,QAAQA,SAASS,MAAME;MAAO;IAAE;AAC5G,WAAO;MAACa;SAAcR;;EACxB;;;;;EAMA,MAAgBH,uBAA8D;AAE5E,UAAMY,OAAyBC,UAC7B,KAAKvC,QAAQwC,cAAcC,WAC3B,MAAM,GAAGjD,WAAAA,mDAA6D;AAGxE,UAAMkD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK,QAAOE;AAEjB,WAAOC,iBAAiBC,KAAKJ,KAAK,KAAKK,OAAO;EAChD;;;;;EAMA,MAAgBhC,iCAAiC;AAE/C,UAAMuB,OAAyBC,UAC7B,KAAKvC,QAAQwC,cAAc1B,qBAC3B,MAAM,GAAGtB,WAAAA,6DAAuE;AAGlF,UAAMkD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK;AAEV,WAAOM,eAAeF,KAMpBJ,KAAK,KAAKK,OAAO;EACrB;EAEA,MAAgBb,0BAA0BD,IAAkBQ,WAAqE;AAC/H,UAAMQ,eAAeC,qCAAoCjB,IAAI,KAAKhC,eAAe,EAAEmC,KAAI;AACvF,QAAIa,aAAazB,WAAW,EAAG,QAAOoB;AACtC,UAAMO,SAAS,IAAIC,IAAIH,YAAAA;AACvB,UAAMpB,kBAAkB,MAAMY,UAAUY,IAAI;SAAIF;KAAO;AACvD,WAAO;MAAClB;SAAOJ;;EACjB;AACF;","names":["IndexingDiviner","TemporalIndexingDivinerConfigSchema","TemporalIndexingDiviner","IndexingDiviner","configSchemas","TemporalIndexingDivinerConfigSchema","defaultConfigSchema","startHandler","AbstractDiviner","jsonPathToTransformersDictionary","reducePayloads","PayloadDivinerQuerySchema","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isPayloadOfSchemaType","TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","divinerQuerySchema","config","PayloadDivinerQuerySchema","indexQuerySchema","indexSchema","TemporalIndexingDivinerResultIndexSchema","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","defaultValue","destinationField","sourcePathExpression","divineHandler","payloads","queries","filter","isPayloadOfSchemaType","length","Promise","all","map","query","fields","reducePayloads","schemas","sources","PayloadBuilder","schema","build","containsAll","assertEx","exists","isBoundWitness","AbstractDiviner","jsonPathToTransformersDictionary","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isAnyPayload","intraBoundwitnessSchemaCombinations","moduleName","TemporalIndexingDivinerIndexCandidateToIndexDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","assertEx","config","divineHandler","payloads","indexableBoundWitnesses","filter","isBoundWitness","bw","containsAll","payload_schemas","indexablePayloads","p","isIndexablePayload","length","bwDictionary","payloadDictionary","Promise","all","PayloadBuilder","toDataHashMap","validIndexableTuples","bwHash","entries","combinations","intraBoundwitnessSchemaCombinations","combination","map","hash","exists","push","indexes","sourcePayloadHashes","sourcePayloads","indexFields","flatMap","payload","transformers","schema","transform","sources","TemporalIndexingDivinerResultIndexSchema","fields","assign","build","flat","x","isAnyPayload","includes","isIndexableSchema","AbstractDiviner","isPayloadDivinerQueryPayload","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","defaultConfigSchema","labels","divineHandler","payloads","responses","filter","p","isPayloadDivinerQueryPayload","Promise","resolve","assertEx","exists","ArchivistWrapper","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","DivinerWrapper","isModuleState","ModuleStateSchema","PayloadBuilder","intraBoundwitnessSchemaCombinations","TimestampSchema","order","moduleName","TemporalIndexingDivinerStateToIndexCandidateDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","defaultConfigSchema","labels","payloadDivinerLimit","config","payload_schemas","schemas","filter","TimestampSchema","divineHandler","payloads","lastState","find","isModuleState","schema","ModuleStateSchema","state","offset","boundWitnessDiviner","getBoundWitnessDivinerForStore","query","PayloadBuilder","BoundWitnessDivinerQuerySchema","fields","limit","build","batch","divine","length","sourceArchivist","getArchivistForStore","bws","isBoundWitness","indexCandidates","Promise","all","map","bw","getPayloadsInBoundWitness","exists","flat","nextState","name","assertEx","payloadStore","archivist","mod","resolve","undefined","ArchivistWrapper","wrap","account","DivinerWrapper","combinations","intraBoundwitnessSchemaCombinations","hashes","Set","get"]}
+{"version":3,"sources":["../../src/Diviner.ts","../../src/DivinerQueryToIndexQueryDiviner/Diviner.ts","../../src/IndexCandidateToIndexDiviner/Diviner.ts","../../src/IndexQueryResponseToDivinerQueryResponseDiviner/Diviner.ts","../../src/StateToIndexCandidateDiviner/Diviner.ts"],"sourcesContent":["import { IndexingDiviner } from '@xyo-network/diviner-indexing-memory'\nimport type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport type { TemporalIndexingDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\nexport class TemporalIndexingDiviner<\n  TParams extends TemporalIndexingDivinerParams = TemporalIndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends IndexingDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerConfigSchema\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    return true\n  }\n}\n","import type { Hash } from '@xylabs/hex'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary, reducePayloads } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport type { TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { isPayloadOfSchemaType } from '@xyo-network/payload-model'\n// TODO: Inherit from JsonPathAggregateDiviner\n/**\n * A diviner that converts diviner query to index query\n */\nexport class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<\n  TParams extends TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas = [...super.configSchemas, TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema]\n  static override readonly defaultConfigSchema = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'divinerQueryToIndexQueryDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * The schema of the diviner query payloads\n   */\n  protected get divinerQuerySchema(): string {\n    return this.config.divinerQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index query payloads\n   */\n  protected get indexQuerySchema(): string {\n    return this.config.indexQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index payloads\n   */\n  protected get indexSchema(): string {\n    return this.config.indexSchema ?? TemporalIndexingDivinerResultIndexSchema\n  }\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return (\n      this.config?.schemaTransforms ?? {\n        [this.divinerQuerySchema]: [\n          {\n            defaultValue: 1,\n            destinationField: 'limit',\n            sourcePathExpression: '$.limit',\n          },\n          {\n            defaultValue: 0,\n            destinationField: 'offset',\n            sourcePathExpression: '$.offset',\n          },\n          {\n            defaultValue: 'desc',\n            destinationField: 'order',\n            sourcePathExpression: '$.order',\n          },\n        ],\n      }\n    )\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    const queries = payloads.filter(isPayloadOfSchemaType<PayloadDivinerQueryPayload>(this.divinerQuerySchema))\n    if (queries.length > 0) {\n      return await Promise.all(\n        queries.map(async (query) => {\n          const fields = await reducePayloads<PayloadDivinerQueryPayload & { sources?: Hash[] }>(\n            [query],\n            this.payloadTransformers,\n            this.indexQuerySchema,\n          )\n          // TODO: Make index schema configurable\n          fields.schemas = [this.indexSchema]\n          // TODO: Make sources not need to be deleted\n          delete fields.sources\n          // TODO: Add support for additional filters\n          return await new PayloadBuilder<Payload>({ schema: this.indexQuerySchema }).fields(fields).build()\n        }),\n      )\n    }\n    return []\n  }\n}\n","import { containsAll } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n  TemporalIndexingDivinerResultIndex,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\nimport { isAnyPayload } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\n\ntype IndexableHashes = [Hash, ...Hash[]]\n\nconst moduleName = 'TemporalIndexingDivinerIndexCandidateToIndexDiviner'\n\n/**\n * Diviner which transforms index candidates to indexes using JSON Path to map\n * source properties in the supplied payloads to destination fields in the\n * resultant index\n */\nexport class TemporalIndexingDivinerIndexCandidateToIndexDiviner<\n  TParams extends TemporalIndexingDivinerIndexCandidateToIndexDivinerParams = TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexCandidateToIndexDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return assertEx(this.config?.schemaTransforms, () => `${moduleName}: Missing config.schemaTransforms section`)\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // If the Bound Witness does not contain all the required schemas do not index it\n    const indexableBoundWitnesses = payloads\n      .filter(isBoundWitness)\n      .filter(bw => containsAll(bw.payload_schemas, this.indexableSchemas))\n    // If the Payload is not one of the indexable schemas do not index it\n    const indexablePayloads = payloads.filter(p => this.isIndexablePayload(p))\n    // If there is nothing to index, return an empty array\n    if (indexableBoundWitnesses.length === 0 || indexablePayloads.length === 0) return []\n    // Hash all the indexable data once\n    const [bwDictionary, payloadDictionary] = await Promise.all([\n      PayloadBuilder.toDataHashMap(indexableBoundWitnesses),\n      PayloadBuilder.toDataHashMap(indexablePayloads),\n    ])\n\n    // Initialize the array for validIndexableTuples outside of the loop\n    const validIndexableTuples: IndexableHashes[] = []\n\n    // Iterate over each entry in bwDictionary\n    for (const [bwHash, bw] of Object.entries(bwDictionary) as [Hash, BoundWitness][]) {\n      // Find the combinations of payloads that satisfy the required schemas\n      const combinations = intraBoundwitnessSchemaCombinations(bw, this.indexableSchemas)\n\n      // Iterate over each combination\n      for (const combination of combinations) {\n        const indexablePayloads = combination.map(hash => payloadDictionary[hash]).filter(exists)\n\n        // If we found the right amount of indexable payloads (of the correct schema as checked\n        // above) in this BW, then index it\n        if (indexablePayloads.length === this.indexableSchemas.length) {\n          validIndexableTuples.push([bwHash, ...combination])\n        }\n      }\n    }\n\n    // Create the indexes from the tuples\n    const indexes = await Promise.all(\n      validIndexableTuples.map<Promise<TemporalIndexingDivinerResultIndex>>(async ([bwHash, ...sourcePayloadHashes]) => {\n        const sourcePayloads = sourcePayloadHashes.map(hash => payloadDictionary[hash])\n        // Use the payload transformers to convert the fields from the source payloads to the destination fields\n        const indexFields = sourcePayloads.flatMap((payload) => {\n          // Find the transformers for this payload\n          const transformers = this.payloadTransformers[payload.schema]\n          // If transformers exist, apply them to the payload otherwise return an empty array\n          return transformers ? transformers.map(transform => transform(payload)) : []\n        })\n        // Include all the sources for reference\n        const sources: string[] = [bwHash, ...sourcePayloadHashes]\n        // Build and return the index\n        return await new PayloadBuilder<TemporalIndexingDivinerResultIndex>({ schema: TemporalIndexingDivinerResultIndexSchema })\n          .fields(Object.assign({ sources }, ...indexFields))\n          .build()\n      }),\n    )\n    return indexes.flat()\n  }\n\n  /**\n   * Identifies if a payload is one that is indexed by this diviner\n   * @param x The candidate payload\n   * @returns True if the payload is one indexed by this diviner, false otherwise\n   */\n  protected isIndexablePayload = (x: unknown) => {\n    return isAnyPayload(x) && this.indexableSchemas.includes(x?.schema)\n  }\n\n  /**\n   * Identifies if a schema is one that is indexed by this diviner\n   * @param schema The candidate schema\n   * @returns True if this schema is one indexed by this diviner, false otherwise\n   */\n  protected isIndexableSchema = (schema?: string | null) => {\n    return typeof schema === 'string' ? this.indexableSchemas.includes(schema) : false\n  }\n}\n","import { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\n/**\n * Transforms an ImageThumbnailIndex response into an ImageThumbnailResponse response\n */\nexport class TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner extends AbstractDiviner {\n  static override readonly configSchemas: Schema[] = [\n    ...super.configSchemas,\n    TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema,\n  ]\n\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexQueryResponseToDivinerQueryResponseDiviner',\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // NOTE: We're not doing anything with the query payloads but some diviners\n    // might want to use this to transform from the query to the response (for example\n    // if we use a plaintext value in the query to generate a hash key in the index)\n    // const queries = payloads.filter(isPayloadDivinerQueryPayload)\n    const responses = payloads.filter(p => !isPayloadDivinerQueryPayload(p))\n    return await Promise.resolve(responses)\n  }\n}\n","import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hex } from '@xylabs/hex'\nimport type { ArchivistInstance } from '@xyo-network/archivist-model'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport type { BoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-abstract'\nimport type { BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'\nimport type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport type {\n  Labels, ModuleIdentifier, ModuleState,\n} from '@xyo-network/module-model'\nimport { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema,\n  WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport { StorageMetaConstants } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\nimport type { TimeStamp } from '@xyo-network/witness-timestamp'\nimport { TimestampSchema } from '@xyo-network/witness-timestamp'\n\n/**\n * All Payload types involved in index candidates for indexing\n */\nexport type IndexCandidate = BoundWitness | Payload | TimeStamp\n\n/**\n * The response from the TemporalStateToIndexCandidateDiviner\n */\nexport type TemporalStateToIndexCandidateDivinerResponse = [\n  /**\n   * The next state of the diviner\n   */\n  nextState: ModuleState<IndexingDivinerState>,\n  /**\n   * The index candidates\n   */\n  ...IndexCandidate[],\n]\n\n/**\n * The default order to search Bound Witnesses to identify index candidates\n */\nconst order = 'asc'\n\n/**\n * The name of the module (for logging purposes)\n */\nconst moduleName = 'TemporalIndexingDivinerStateToIndexCandidateDiviner'\n\n/**\n * Transforms candidates for image thumbnail indexing into their indexed representation\n */\nexport class TemporalIndexingDivinerStateToIndexCandidateDiviner<\n  TParams extends TemporalIndexingDivinerStateToIndexCandidateDivinerParams = TemporalIndexingDivinerStateToIndexCandidateDivinerParams,\n> extends AbstractDiviner<TParams, Payload, ModuleState | IndexCandidate> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'stateToIndexCandidateDiviner',\n  }\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1000\n  }\n\n  /**\n   * The required payload_schemas within BoundWitnesses to identify index candidates\n   */\n  protected get payload_schemas(): string[] {\n    const schemas = this.config.filter?.payload_schemas\n    return [TimestampSchema, ...(schemas ?? [])]\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<[ModuleState, ...IndexCandidate[]]> {\n    // Retrieve the last state from what was passed in\n    const lastState = payloads.find(isModuleState<IndexingDivinerState>)\n    // If there is no last state, start from the beginning\n    if (!lastState) return [{ schema: ModuleStateSchema, state: { cursor: StorageMetaConstants.minLocalSequence } }]\n    // Otherwise, get the last offset\n    const { cursor } = lastState.state\n    // Get next batch of results starting from the offset\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore()\n    if (!boundWitnessDiviner) return [lastState]\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit, cursor: cursor as Hex, order, payload_schemas: this.payload_schemas,\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return [lastState]\n    // Get source data\n    const sourceArchivist = await this.getArchivistForStore()\n    if (!sourceArchivist) return [lastState]\n    const bws = batch.filter(isBoundWitness)\n    const indexCandidates: IndexCandidate[] = (await Promise.all(bws.map(bw => this.getPayloadsInBoundWitness(bw, sourceArchivist))))\n      .filter(exists)\n      .flat()\n    const nextState = { schema: ModuleStateSchema, state: { ...lastState.state, cursor: batch.at(-1)?._sequence } }\n    return [nextState, ...indexCandidates]\n  }\n\n  /**\n   * Retrieves the archivist for the payloadStore\n   * @returns The archivist for the payloadStore or undefined if not resolvable\n   */\n  protected async getArchivistForStore(): Promise<ArchivistWrapper | undefined> {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.archivist,\n      () => `${moduleName}: Config for payloadStore.archivist not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return undefined\n    // Return the wrapped archivist\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the payloadStore\n   * @returns The BoundWitness Diviner for the payloadStore or undefined if not resolvable\n   */\n  protected async getBoundWitnessDivinerForStore() {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.boundWitnessDiviner,\n      () => `${moduleName}: Config for payloadStore.boundWitnessDiviner not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return\n    // Return the wrapped diviner\n    return DivinerWrapper.wrap<\n      DivinerWrapper<\n        BoundWitnessDiviner<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, BoundWitness>,\n        BoundWitnessDivinerQueryPayload,\n        WithStorageMeta<BoundWitness>\n      >\n    >(mod, this.account)\n  }\n\n  protected async getPayloadsInBoundWitness(bw: BoundWitness, archivist: ArchivistInstance): Promise<IndexCandidate[] | undefined> {\n    const combinations = intraBoundwitnessSchemaCombinations(bw, this.payload_schemas).flat()\n    if (combinations.length === 0) return undefined\n    const hashes = new Set(combinations)\n    const indexCandidates = await archivist.get([...hashes])\n    return [bw, ...indexCandidates]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,uBAAuB;AAGhC,SAASC,2CAA2C;AAG7C,IAAMC,0BAAN,cASGC,gBAAAA;EAfV,OAeUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAEvD,MAAyBE,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,WAAO;EACT;AACF;;;ACtBA,SAASC,uBAAuB;AAChC,SAASC,kCAAkCC,sBAAsB;AAGjE,SAASC,iCAAiC;AAE1C,SACEC,oEACAC,gDACK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,6BAA6B;AAK/B,IAAMC,yDAAN,cAEGC,gBAAAA;EApBV,OAoBUA;;;EACR,OAAyBC,gBAAgB;OAAI,MAAMA;IAAeC;;EAClE,OAAyBC,sBAAsBD;EAC/C,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,qBAA6B;AACzC,WAAO,KAAKC,OAAOD,sBAAsBE;EAC3C;;;;EAKA,IAAcC,mBAA2B;AACvC,WAAO,KAAKF,OAAOE,oBAAoBD;EACzC;;;;EAKA,IAAcE,cAAsB;AAClC,WAAO,KAAKH,OAAOG,eAAeC;EACpC;;;;EAKA,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKR,kBAAmB,MAAKA,oBAAoBS,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKX;EACd;;;;;EAMA,IAAcY,sBAA6D;AACzE,QAAI,CAAC,KAAKX,qBAAsB,MAAKA,uBAAuBY,iCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKV;EACd;;;;;EAMA,IAAcU,mBAAmE;AAC/E,WACE,KAAKR,QAAQQ,oBAAoB;MAC/B,CAAC,KAAKT,kBAAkB,GAAG;QACzB;UACEY,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;;IAEJ;EAEJ;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AACnF,UAAMC,UAAUD,SAASE,OAAOC,sBAAkD,KAAKnB,kBAAkB,CAAA;AACzG,QAAIiB,QAAQG,SAAS,GAAG;AACtB,aAAO,MAAMC,QAAQC,IACnBL,QAAQM,IAAI,OAAOC,UAAAA;AACjB,cAAMC,SAAS,MAAMC,eACnB;UAACF;WACD,KAAKd,qBACL,KAAKP,gBAAgB;AAGvBsB,eAAOE,UAAU;UAAC,KAAKvB;;AAEvB,eAAOqB,OAAOG;AAEd,eAAO,MAAM,IAAIC,eAAwB;UAAEC,QAAQ,KAAK3B;QAAiB,CAAA,EAAGsB,OAAOA,MAAAA,EAAQM,MAAK;MAClG,CAAA,CAAA;IAEJ;AACA,WAAO,CAAA;EACT;AACF;;;ACvHA,SAASC,mBAAmB;AAC5B,SAASC,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,sBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAChC,SAASC,oCAAAA,yCAAwC;AAMjD,SACEC,iEACAC,4CAAAA,iDACK;AAEP,SAASC,kBAAAA,uBAAsB;AAE/B,SAASC,oBAAoB;AAC7B,SAASC,2CAA2C;AAIpD,IAAMC,aAAa;AAOZ,IAAMC,sDAAN,cAEGC,iBAAAA;EAlCV,OAkCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKF,kBAAmB,MAAKA,oBAAoBG,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKL;EACd;;;;;EAMA,IAAcM,sBAA6D;AACzE,QAAI,CAAC,KAAKL,qBAAsB,MAAKA,uBAAuBM,kCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKJ;EACd;;;;;EAMA,IAAcI,mBAAmE;AAC/E,WAAOG,SAAS,KAAKC,QAAQJ,kBAAkB,MAAM,GAAGZ,UAAAA,2CAAqD;EAC/G;EAEA,MAAyBiB,cAAcC,WAAsB,CAAA,GAAwB;AAEnF,UAAMC,0BAA0BD,SAC7BE,OAAOC,cAAAA,EACPD,OAAOE,CAAAA,OAAMC,YAAYD,GAAGE,iBAAiB,KAAKf,gBAAgB,CAAA;AAErE,UAAMgB,oBAAoBP,SAASE,OAAOM,CAAAA,MAAK,KAAKC,mBAAmBD,CAAAA,CAAAA;AAEvE,QAAIP,wBAAwBS,WAAW,KAAKH,kBAAkBG,WAAW,EAAG,QAAO,CAAA;AAEnF,UAAM,CAACC,cAAcC,iBAAAA,IAAqB,MAAMC,QAAQC,IAAI;MAC1DC,gBAAeC,cAAcf,uBAAAA;MAC7Bc,gBAAeC,cAAcT,iBAAAA;KAC9B;AAGD,UAAMU,uBAA0C,CAAA;AAGhD,eAAW,CAACC,QAAQd,EAAAA,KAAOZ,OAAO2B,QAAQR,YAAAA,GAAyC;AAEjF,YAAMS,eAAeC,oCAAoCjB,IAAI,KAAKb,gBAAgB;AAGlF,iBAAW+B,eAAeF,cAAc;AACtC,cAAMb,qBAAoBe,YAAYC,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK,EAAEtB,OAAOuB,MAAAA;AAIlF,YAAIlB,mBAAkBG,WAAW,KAAKnB,iBAAiBmB,QAAQ;AAC7DO,+BAAqBS,KAAK;YAACR;eAAWI;WAAY;QACpD;MACF;IACF;AAGA,UAAMK,UAAU,MAAMd,QAAQC,IAC5BG,qBAAqBM,IAAiD,OAAO,CAACL,QAAWU,sBAAAA,MAAoB;AAC3G,YAAMC,iBAAiBD,oBAAoBL,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK;AAE9E,YAAMM,cAAcD,eAAeE,QAAQ,CAACC,YAAAA;AAE1C,cAAMC,eAAe,KAAKtC,oBAAoBqC,QAAQE,MAAM;AAE5D,eAAOD,eAAeA,aAAaV,IAAIY,CAAAA,cAAaA,UAAUH,OAAAA,CAAAA,IAAY,CAAA;MAC5E,CAAA;AAEA,YAAMI,UAAoB;QAAClB;WAAWU;;AAEtC,aAAO,MAAM,IAAIb,gBAAmD;QAAEmB,QAAQG;MAAyC,CAAA,EACpHC,OAAO9C,OAAO+C,OAAO;QAAEH;MAAQ,GAAA,GAAMN,WAAAA,CAAAA,EACrCU,MAAK;IACV,CAAA,CAAA;AAEF,WAAOb,QAAQc,KAAI;EACrB;;;;;;EAOUhC,qBAAqB,wBAACiC,MAAAA;AAC9B,WAAOC,aAAaD,CAAAA,KAAM,KAAKnD,iBAAiBqD,SAASF,GAAGR,MAAAA;EAC9D,GAF+B;;;;;;EASrBW,oBAAoB,wBAACX,WAAAA;AAC7B,WAAO,OAAOA,WAAW,WAAW,KAAK3C,iBAAiBqD,SAASV,MAAAA,IAAU;EAC/E,GAF8B;AAGhC;;;AChJA,SAASY,mBAAAA,wBAAuB;AAChC,SAASC,oCAAoC;AAC7C,SAASC,0FAA0F;AAO5F,IAAMC,yEAAN,cAAqFC,iBAAAA;EAT5F,OAS4FA;;;EAC1F,OAAyBC,gBAA0B;OAC9C,MAAMA;IACTC;;EAGF,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AAKnF,UAAMC,YAAYD,SAASE,OAAOC,CAAAA,MAAK,CAACC,6BAA6BD,CAAAA,CAAAA;AACrE,WAAO,MAAME,QAAQC,QAAQL,SAAAA;EAC/B;AACF;;;AC7BA,SAASM,YAAAA,iBAAgB;AACzB,SAASC,UAAAA,eAAc;AAGvB,SAASC,wBAAwB;AAEjC,SAASC,kBAAAA,uBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAGhC,SAASC,sCAAsC;AAG/C,SAASC,uEAAuE;AAChF,SAASC,sBAAsB;AAI/B,SAASC,eAAeC,yBAAyB;AACjD,SAASC,kBAAAA,uBAAsB;AAK/B,SAASC,4BAA4B;AACrC,SAASC,uCAAAA,4CAA2C;AAEpD,SAASC,uBAAuB;AAwBhC,IAAMC,QAAQ;AAKd,IAAMC,cAAa;AAKZ,IAAMC,sDAAN,cAEGC,iBAAAA;EA/DV,OA+DUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;;;;EAKA,IAAcE,kBAA4B;AACxC,UAAMC,UAAU,KAAKF,OAAOG,QAAQF;AACpC,WAAO;MAACG;SAAqBF,WAAW,CAAA;;EAC1C;EAEA,MAAyBG,cAAcC,WAAsB,CAAA,GAAiD;AAE5G,UAAMC,YAAYD,SAASE,KAAKC,aAAAA;AAEhC,QAAI,CAACF,UAAW,QAAO;MAAC;QAAEG,QAAQC;QAAmBC,OAAO;UAAEC,QAAQC,qBAAqBC;QAAiB;MAAE;;AAE9G,UAAM,EAAEF,OAAM,IAAKN,UAAUK;AAE7B,UAAMI,sBAAsB,MAAM,KAAKC,+BAA8B;AACrE,QAAI,CAACD,oBAAqB,QAAO;MAACT;;AAClC,UAAMW,QAAQ,IAAIC,gBAAgD;MAAET,QAAQU;IAA+B,CAAA,EACxGC,OAAO;MACNC,OAAO,KAAKvB;MAAqBc;MAAuBtB;MAAOU,iBAAiB,KAAKA;IACvF,CAAA,EACCsB,MAAK;AACR,UAAMC,QAAQ,MAAMR,oBAAoBS,OAAO;MAACP;KAAM;AACtD,QAAIM,MAAME,WAAW,EAAG,QAAO;MAACnB;;AAEhC,UAAMoB,kBAAkB,MAAM,KAAKC,qBAAoB;AACvD,QAAI,CAACD,gBAAiB,QAAO;MAACpB;;AAC9B,UAAMsB,MAAML,MAAMrB,OAAO2B,eAAAA;AACzB,UAAMC,mBAAqC,MAAMC,QAAQC,IAAIJ,IAAIK,IAAIC,CAAAA,OAAM,KAAKC,0BAA0BD,IAAIR,eAAAA,CAAAA,CAAAA,GAC3GxB,OAAOkC,OAAAA,EACPC,KAAI;AACP,UAAMC,YAAY;MAAE7B,QAAQC;MAAmBC,OAAO;QAAE,GAAGL,UAAUK;QAAOC,QAAQW,MAAMgB,GAAG,EAAC,GAAIC;MAAU;IAAE;AAC9G,WAAO;MAACF;SAAcR;;EACxB;;;;;EAMA,MAAgBH,uBAA8D;AAE5E,UAAMc,OAAyBC,UAC7B,KAAK3C,QAAQ4C,cAAcC,WAC3B,MAAM,GAAGrD,WAAAA,mDAA6D;AAGxE,UAAMsD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK,QAAOE;AAEjB,WAAOC,iBAAiBC,KAAKJ,KAAK,KAAKK,OAAO;EAChD;;;;;EAMA,MAAgBlC,iCAAiC;AAE/C,UAAMyB,OAAyBC,UAC7B,KAAK3C,QAAQ4C,cAAc5B,qBAC3B,MAAM,GAAGxB,WAAAA,6DAAuE;AAGlF,UAAMsD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK;AAEV,WAAOM,eAAeF,KAMpBJ,KAAK,KAAKK,OAAO;EACrB;EAEA,MAAgBf,0BAA0BD,IAAkBU,WAAqE;AAC/H,UAAMQ,eAAeC,qCAAoCnB,IAAI,KAAKlC,eAAe,EAAEqC,KAAI;AACvF,QAAIe,aAAa3B,WAAW,EAAG,QAAOsB;AACtC,UAAMO,SAAS,IAAIC,IAAIH,YAAAA;AACvB,UAAMtB,kBAAkB,MAAMc,UAAUY,IAAI;SAAIF;KAAO;AACvD,WAAO;MAACpB;SAAOJ;;EACjB;AACF;","names":["IndexingDiviner","TemporalIndexingDivinerConfigSchema","TemporalIndexingDiviner","IndexingDiviner","configSchemas","TemporalIndexingDivinerConfigSchema","defaultConfigSchema","startHandler","AbstractDiviner","jsonPathToTransformersDictionary","reducePayloads","PayloadDivinerQuerySchema","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isPayloadOfSchemaType","TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","divinerQuerySchema","config","PayloadDivinerQuerySchema","indexQuerySchema","indexSchema","TemporalIndexingDivinerResultIndexSchema","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","defaultValue","destinationField","sourcePathExpression","divineHandler","payloads","queries","filter","isPayloadOfSchemaType","length","Promise","all","map","query","fields","reducePayloads","schemas","sources","PayloadBuilder","schema","build","containsAll","assertEx","exists","isBoundWitness","AbstractDiviner","jsonPathToTransformersDictionary","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isAnyPayload","intraBoundwitnessSchemaCombinations","moduleName","TemporalIndexingDivinerIndexCandidateToIndexDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","assertEx","config","divineHandler","payloads","indexableBoundWitnesses","filter","isBoundWitness","bw","containsAll","payload_schemas","indexablePayloads","p","isIndexablePayload","length","bwDictionary","payloadDictionary","Promise","all","PayloadBuilder","toDataHashMap","validIndexableTuples","bwHash","entries","combinations","intraBoundwitnessSchemaCombinations","combination","map","hash","exists","push","indexes","sourcePayloadHashes","sourcePayloads","indexFields","flatMap","payload","transformers","schema","transform","sources","TemporalIndexingDivinerResultIndexSchema","fields","assign","build","flat","x","isAnyPayload","includes","isIndexableSchema","AbstractDiviner","isPayloadDivinerQueryPayload","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","defaultConfigSchema","labels","divineHandler","payloads","responses","filter","p","isPayloadDivinerQueryPayload","Promise","resolve","assertEx","exists","ArchivistWrapper","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","DivinerWrapper","isModuleState","ModuleStateSchema","PayloadBuilder","StorageMetaConstants","intraBoundwitnessSchemaCombinations","TimestampSchema","order","moduleName","TemporalIndexingDivinerStateToIndexCandidateDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","defaultConfigSchema","labels","payloadDivinerLimit","config","payload_schemas","schemas","filter","TimestampSchema","divineHandler","payloads","lastState","find","isModuleState","schema","ModuleStateSchema","state","cursor","StorageMetaConstants","minLocalSequence","boundWitnessDiviner","getBoundWitnessDivinerForStore","query","PayloadBuilder","BoundWitnessDivinerQuerySchema","fields","limit","build","batch","divine","length","sourceArchivist","getArchivistForStore","bws","isBoundWitness","indexCandidates","Promise","all","map","bw","getPayloadsInBoundWitness","exists","flat","nextState","at","_sequence","name","assertEx","payloadStore","archivist","mod","resolve","undefined","ArchivistWrapper","wrap","account","DivinerWrapper","combinations","intraBoundwitnessSchemaCombinations","hashes","Set","get"]}

package/dist/node/StateToIndexCandidateDiviner/Diviner.d.ts

@@ -8,7 +8,7 @@ import type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model';
 import type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model';
 import { DivinerWrapper } from '@xyo-network/diviner-wrapper';
 import type { Labels, ModuleState } from '@xyo-network/module-model';
-import type { Payload, Schema } from '@xyo-network/payload-model';
+import type { Payload, Schema, WithStorageMeta } from '@xyo-network/payload-model';
 import type { TimeStamp } from '@xyo-network/witness-timestamp';
 /**
  * All Payload types involved in index candidates for indexing
@@ -101,7 +101,7 @@ export declare class TemporalIndexingDivinerStateToIndexCandidateDiviner<TParams
         schema: BoundWitnessDivinerQuerySchema;
     } & import("@xyo-network/diviner-boundwitness-model").BoundWitnessDivinerPredicate & import("@xyo-network/payload-model").QueryFields, "schema"> & {
         schema: "network.xyo.diviner.boundwitness.query";
-    }, BoundWitness> | undefined>;
+    }, WithStorageMeta<BoundWitness>> | undefined>;
     protected getPayloadsInBoundWitness(bw: BoundWitness, archivist: ArchivistInstance): Promise<IndexCandidate[] | undefined>;
 }
 //# sourceMappingURL=Diviner.d.ts.map

package/dist/node/StateToIndexCandidateDiviner/Diviner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/StateToIndexCandidateDiviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAA;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAEnE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4CAA4C,CAAA;AAErF,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAA;AACxF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAA;AAC/E,OAAO,KAAK,EAAE,yDAAyD,EAAE,MAAM,8CAA8C,CAAA;AAE7H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EACV,MAAM,EAAoB,WAAW,EACtC,MAAM,2BAA2B,CAAA;AAGlC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAA;AAEjE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAA;AAG/D;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,CAAA;AAE/D;;GAEG;AACH,MAAM,MAAM,4CAA4C,GAAG;IACzD;;OAEG;IACH,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;IAC5C;;OAEG;IACH,GAAG,cAAc,EAAE;CACpB,CAAA;AAYD;;GAEG;AACH,qBAAa,mDAAmD,CAC9D,OAAO,SAAS,yDAAyD,GAAG,yDAAyD,CACrI,SAAQ,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,GAAG,cAAc,CAAC;IACvE,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA4F;IAC5I,gBAAyB,mBAAmB,EAAE,MAAM,CAAkE;IACtH,OAAgB,MAAM,EAAE,MAAM,CAG7B;IAED,IAAI,mBAAmB,WAEtB;IAED;;OAEG;IACH,SAAS,KAAK,eAAe,IAAI,MAAM,EAAE,CAGxC;cAEwB,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,CAAC,WAAW,EAAE,GAAG,cAAc,EAAE,CAAC,CAAC;IA4B7G;;;OAGG;cACa,oBAAoB,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAa7E;;;OAGG;cACa,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAmB9B,yBAAyB,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,EAAE,GAAG,SAAS,CAAC;CAOjI"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/StateToIndexCandidateDiviner/Diviner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAA;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAEnE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4CAA4C,CAAA;AAErF,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAA;AACxF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAA;AAC/E,OAAO,KAAK,EAAE,yDAAyD,EAAE,MAAM,8CAA8C,CAAA;AAE7H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EACV,MAAM,EAAoB,WAAW,EACtC,MAAM,2BAA2B,CAAA;AAGlC,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EACf,eAAe,EAChB,MAAM,4BAA4B,CAAA;AAGnC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAA;AAG/D;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,CAAA;AAE/D;;GAEG;AACH,MAAM,MAAM,4CAA4C,GAAG;IACzD;;OAEG;IACH,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;IAC5C;;OAEG;IACH,GAAG,cAAc,EAAE;CACpB,CAAA;AAYD;;GAEG;AACH,qBAAa,mDAAmD,CAC9D,OAAO,SAAS,yDAAyD,GAAG,yDAAyD,CACrI,SAAQ,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,GAAG,cAAc,CAAC;IACvE,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA4F;IAC5I,gBAAyB,mBAAmB,EAAE,MAAM,CAAkE;IACtH,OAAgB,MAAM,EAAE,MAAM,CAG7B;IAED,IAAI,mBAAmB,WAEtB;IAED;;OAEG;IACH,SAAS,KAAK,eAAe,IAAI,MAAM,EAAE,CAGxC;cAEwB,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,CAAC,WAAW,EAAE,GAAG,cAAc,EAAE,CAAC,CAAC;IA4B7G;;;OAGG;cACa,oBAAoB,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAa7E;;;OAGG;cACa,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAmB9B,yBAAyB,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,EAAE,GAAG,SAAS,CAAC;CAOjI"}

package/dist/node/index.mjs

@@ -259,6 +259,7 @@ import { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from
 import { DivinerWrapper } from "@xyo-network/diviner-wrapper";
 import { isModuleState, ModuleStateSchema } from "@xyo-network/module-model";
 import { PayloadBuilder as PayloadBuilder3 } from "@xyo-network/payload-builder";
+import { StorageMetaConstants } from "@xyo-network/payload-model";
 import { intraBoundwitnessSchemaCombinations as intraBoundwitnessSchemaCombinations2 } from "@xyo-network/payload-utils";
 import { TimestampSchema } from "@xyo-network/witness-timestamp";
 var order = "asc";
@@ -295,20 +296,20 @@ var TemporalIndexingDivinerStateToIndexCandidateDiviner = class extends Abstract
       {
         schema: ModuleStateSchema,
         state: {
-          offset: 0
+          cursor: StorageMetaConstants.minLocalSequence
         }
       }
     ];
-    const { offset } = lastState.state;
+    const { cursor } = lastState.state;
     const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore();
     if (!boundWitnessDiviner) return [
       lastState
     ];
-    const query = await new PayloadBuilder3({
+    const query = new PayloadBuilder3({
       schema: BoundWitnessDivinerQuerySchema
     }).fields({
       limit: this.payloadDivinerLimit,
-      offset,
+      cursor,
       order,
       payload_schemas: this.payload_schemas
     }).build();
@@ -328,7 +329,7 @@ var TemporalIndexingDivinerStateToIndexCandidateDiviner = class extends Abstract
       schema: ModuleStateSchema,
       state: {
         ...lastState.state,
-        offset: offset + batch.length
+        cursor: batch.at(-1)?._sequence
       }
     };
     return [

package/dist/node/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner.ts","../../src/DivinerQueryToIndexQueryDiviner/Diviner.ts","../../src/IndexCandidateToIndexDiviner/Diviner.ts","../../src/IndexQueryResponseToDivinerQueryResponseDiviner/Diviner.ts","../../src/StateToIndexCandidateDiviner/Diviner.ts"],"sourcesContent":["import { IndexingDiviner } from '@xyo-network/diviner-indexing-memory'\nimport type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport type { TemporalIndexingDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\nexport class TemporalIndexingDiviner<\n  TParams extends TemporalIndexingDivinerParams = TemporalIndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends IndexingDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerConfigSchema\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    return true\n  }\n}\n","import type { Hash } from '@xylabs/hex'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary, reducePayloads } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport type { TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { isPayloadOfSchemaType } from '@xyo-network/payload-model'\n// TODO: Inherit from JsonPathAggregateDiviner\n/**\n * A diviner that converts diviner query to index query\n */\nexport class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<\n  TParams extends TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas = [...super.configSchemas, TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema]\n  static override readonly defaultConfigSchema = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'divinerQueryToIndexQueryDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * The schema of the diviner query payloads\n   */\n  protected get divinerQuerySchema(): string {\n    return this.config.divinerQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index query payloads\n   */\n  protected get indexQuerySchema(): string {\n    return this.config.indexQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index payloads\n   */\n  protected get indexSchema(): string {\n    return this.config.indexSchema ?? TemporalIndexingDivinerResultIndexSchema\n  }\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return (\n      this.config?.schemaTransforms ?? {\n        [this.divinerQuerySchema]: [\n          {\n            defaultValue: 1,\n            destinationField: 'limit',\n            sourcePathExpression: '$.limit',\n          },\n          {\n            defaultValue: 0,\n            destinationField: 'offset',\n            sourcePathExpression: '$.offset',\n          },\n          {\n            defaultValue: 'desc',\n            destinationField: 'order',\n            sourcePathExpression: '$.order',\n          },\n        ],\n      }\n    )\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    const queries = payloads.filter(isPayloadOfSchemaType<PayloadDivinerQueryPayload>(this.divinerQuerySchema))\n    if (queries.length > 0) {\n      return await Promise.all(\n        queries.map(async (query) => {\n          const fields = await reducePayloads<PayloadDivinerQueryPayload & { sources?: Hash[] }>(\n            [query],\n            this.payloadTransformers,\n            this.indexQuerySchema,\n          )\n          // TODO: Make index schema configurable\n          fields.schemas = [this.indexSchema]\n          // TODO: Make sources not need to be deleted\n          delete fields.sources\n          // TODO: Add support for additional filters\n          return await new PayloadBuilder<Payload>({ schema: this.indexQuerySchema }).fields(fields).build()\n        }),\n      )\n    }\n    return []\n  }\n}\n","import { containsAll } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n  TemporalIndexingDivinerResultIndex,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\nimport { isAnyPayload } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\n\ntype IndexableHashes = [Hash, ...Hash[]]\n\nconst moduleName = 'TemporalIndexingDivinerIndexCandidateToIndexDiviner'\n\n/**\n * Diviner which transforms index candidates to indexes using JSON Path to map\n * source properties in the supplied payloads to destination fields in the\n * resultant index\n */\nexport class TemporalIndexingDivinerIndexCandidateToIndexDiviner<\n  TParams extends TemporalIndexingDivinerIndexCandidateToIndexDivinerParams = TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexCandidateToIndexDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return assertEx(this.config?.schemaTransforms, () => `${moduleName}: Missing config.schemaTransforms section`)\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // If the Bound Witness does not contain all the required schemas do not index it\n    const indexableBoundWitnesses = payloads\n      .filter(isBoundWitness)\n      .filter(bw => containsAll(bw.payload_schemas, this.indexableSchemas))\n    // If the Payload is not one of the indexable schemas do not index it\n    const indexablePayloads = payloads.filter(p => this.isIndexablePayload(p))\n    // If there is nothing to index, return an empty array\n    if (indexableBoundWitnesses.length === 0 || indexablePayloads.length === 0) return []\n    // Hash all the indexable data once\n    const [bwDictionary, payloadDictionary] = await Promise.all([\n      PayloadBuilder.toDataHashMap(indexableBoundWitnesses),\n      PayloadBuilder.toDataHashMap(indexablePayloads),\n    ])\n\n    // Initialize the array for validIndexableTuples outside of the loop\n    const validIndexableTuples: IndexableHashes[] = []\n\n    // Iterate over each entry in bwDictionary\n    for (const [bwHash, bw] of Object.entries(bwDictionary) as [Hash, BoundWitness][]) {\n      // Find the combinations of payloads that satisfy the required schemas\n      const combinations = intraBoundwitnessSchemaCombinations(bw, this.indexableSchemas)\n\n      // Iterate over each combination\n      for (const combination of combinations) {\n        const indexablePayloads = combination.map(hash => payloadDictionary[hash]).filter(exists)\n\n        // If we found the right amount of indexable payloads (of the correct schema as checked\n        // above) in this BW, then index it\n        if (indexablePayloads.length === this.indexableSchemas.length) {\n          validIndexableTuples.push([bwHash, ...combination])\n        }\n      }\n    }\n\n    // Create the indexes from the tuples\n    const indexes = await Promise.all(\n      validIndexableTuples.map<Promise<TemporalIndexingDivinerResultIndex>>(async ([bwHash, ...sourcePayloadHashes]) => {\n        const sourcePayloads = sourcePayloadHashes.map(hash => payloadDictionary[hash])\n        // Use the payload transformers to convert the fields from the source payloads to the destination fields\n        const indexFields = sourcePayloads.flatMap((payload) => {\n          // Find the transformers for this payload\n          const transformers = this.payloadTransformers[payload.schema]\n          // If transformers exist, apply them to the payload otherwise return an empty array\n          return transformers ? transformers.map(transform => transform(payload)) : []\n        })\n        // Include all the sources for reference\n        const sources: string[] = [bwHash, ...sourcePayloadHashes]\n        // Build and return the index\n        return await new PayloadBuilder<TemporalIndexingDivinerResultIndex>({ schema: TemporalIndexingDivinerResultIndexSchema })\n          .fields(Object.assign({ sources }, ...indexFields))\n          .build()\n      }),\n    )\n    return indexes.flat()\n  }\n\n  /**\n   * Identifies if a payload is one that is indexed by this diviner\n   * @param x The candidate payload\n   * @returns True if the payload is one indexed by this diviner, false otherwise\n   */\n  protected isIndexablePayload = (x: unknown) => {\n    return isAnyPayload(x) && this.indexableSchemas.includes(x?.schema)\n  }\n\n  /**\n   * Identifies if a schema is one that is indexed by this diviner\n   * @param schema The candidate schema\n   * @returns True if this schema is one indexed by this diviner, false otherwise\n   */\n  protected isIndexableSchema = (schema?: string | null) => {\n    return typeof schema === 'string' ? this.indexableSchemas.includes(schema) : false\n  }\n}\n","import { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\n/**\n * Transforms an ImageThumbnailIndex response into an ImageThumbnailResponse response\n */\nexport class TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner extends AbstractDiviner {\n  static override readonly configSchemas: Schema[] = [\n    ...super.configSchemas,\n    TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema,\n  ]\n\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexQueryResponseToDivinerQueryResponseDiviner',\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // NOTE: We're not doing anything with the query payloads but some diviners\n    // might want to use this to transform from the query to the response (for example\n    // if we use a plaintext value in the query to generate a hash key in the index)\n    // const queries = payloads.filter(isPayloadDivinerQueryPayload)\n    const responses = payloads.filter(p => !isPayloadDivinerQueryPayload(p))\n    return await Promise.resolve(responses)\n  }\n}\n","import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { ArchivistInstance } from '@xyo-network/archivist-model'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport type { BoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-abstract'\nimport type { BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'\nimport type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport type {\n  Labels, ModuleIdentifier, ModuleState,\n} from '@xyo-network/module-model'\nimport { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\nimport type { TimeStamp } from '@xyo-network/witness-timestamp'\nimport { TimestampSchema } from '@xyo-network/witness-timestamp'\n\n/**\n * All Payload types involved in index candidates for indexing\n */\nexport type IndexCandidate = BoundWitness | Payload | TimeStamp\n\n/**\n * The response from the TemporalStateToIndexCandidateDiviner\n */\nexport type TemporalStateToIndexCandidateDivinerResponse = [\n  /**\n   * The next state of the diviner\n   */\n  nextState: ModuleState<IndexingDivinerState>,\n  /**\n   * The index candidates\n   */\n  ...IndexCandidate[],\n]\n\n/**\n * The default order to search Bound Witnesses to identify index candidates\n */\nconst order = 'asc'\n\n/**\n * The name of the module (for logging purposes)\n */\nconst moduleName = 'TemporalIndexingDivinerStateToIndexCandidateDiviner'\n\n/**\n * Transforms candidates for image thumbnail indexing into their indexed representation\n */\nexport class TemporalIndexingDivinerStateToIndexCandidateDiviner<\n  TParams extends TemporalIndexingDivinerStateToIndexCandidateDivinerParams = TemporalIndexingDivinerStateToIndexCandidateDivinerParams,\n> extends AbstractDiviner<TParams, Payload, ModuleState | IndexCandidate> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'stateToIndexCandidateDiviner',\n  }\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1000\n  }\n\n  /**\n   * The required payload_schemas within BoundWitnesses to identify index candidates\n   */\n  protected get payload_schemas(): string[] {\n    const schemas = this.config.filter?.payload_schemas\n    return [TimestampSchema, ...(schemas ?? [])]\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<[ModuleState, ...IndexCandidate[]]> {\n    // Retrieve the last state from what was passed in\n    const lastState = payloads.find(isModuleState<IndexingDivinerState>)\n    // If there is no last state, start from the beginning\n    if (!lastState) return [{ schema: ModuleStateSchema, state: { offset: 0 } }]\n    // Otherwise, get the last offset\n    const { offset } = lastState.state\n    // Get next batch of results starting from the offset\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore()\n    if (!boundWitnessDiviner) return [lastState]\n    const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit, offset, order, payload_schemas: this.payload_schemas,\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return [lastState]\n    // Get source data\n    const sourceArchivist = await this.getArchivistForStore()\n    if (!sourceArchivist) return [lastState]\n    const bws = batch.filter(isBoundWitness)\n    const indexCandidates: IndexCandidate[] = (await Promise.all(bws.map(bw => this.getPayloadsInBoundWitness(bw, sourceArchivist))))\n      .filter(exists)\n      .flat()\n    const nextState = { schema: ModuleStateSchema, state: { ...lastState.state, offset: offset + batch.length } }\n    return [nextState, ...indexCandidates]\n  }\n\n  /**\n   * Retrieves the archivist for the payloadStore\n   * @returns The archivist for the payloadStore or undefined if not resolvable\n   */\n  protected async getArchivistForStore(): Promise<ArchivistWrapper | undefined> {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.archivist,\n      () => `${moduleName}: Config for payloadStore.archivist not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return undefined\n    // Return the wrapped archivist\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the payloadStore\n   * @returns The BoundWitness Diviner for the payloadStore or undefined if not resolvable\n   */\n  protected async getBoundWitnessDivinerForStore() {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.boundWitnessDiviner,\n      () => `${moduleName}: Config for payloadStore.boundWitnessDiviner not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return\n    // Return the wrapped diviner\n    return DivinerWrapper.wrap<\n      DivinerWrapper<\n        BoundWitnessDiviner<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, BoundWitness>,\n        BoundWitnessDivinerQueryPayload,\n        BoundWitness\n      >\n    >(mod, this.account)\n  }\n\n  protected async getPayloadsInBoundWitness(bw: BoundWitness, archivist: ArchivistInstance): Promise<IndexCandidate[] | undefined> {\n    const combinations = intraBoundwitnessSchemaCombinations(bw, this.payload_schemas).flat()\n    if (combinations.length === 0) return undefined\n    const hashes = new Set(combinations)\n    const indexCandidates = await archivist.get([...hashes])\n    return [bw, ...indexCandidates]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,uBAAuB;AAGhC,SAASC,2CAA2C;AAG7C,IAAMC,0BAAN,cASGC,gBAAAA;EAfV,OAeUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAEvD,MAAyBE,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,WAAO;EACT;AACF;;;ACtBA,SAASC,uBAAuB;AAChC,SAASC,kCAAkCC,sBAAsB;AAGjE,SAASC,iCAAiC;AAE1C,SACEC,oEACAC,gDACK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,6BAA6B;AAK/B,IAAMC,yDAAN,cAEGC,gBAAAA;EApBV,OAoBUA;;;EACR,OAAyBC,gBAAgB;OAAI,MAAMA;IAAeC;;EAClE,OAAyBC,sBAAsBD;EAC/C,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,qBAA6B;AACzC,WAAO,KAAKC,OAAOD,sBAAsBE;EAC3C;;;;EAKA,IAAcC,mBAA2B;AACvC,WAAO,KAAKF,OAAOE,oBAAoBD;EACzC;;;;EAKA,IAAcE,cAAsB;AAClC,WAAO,KAAKH,OAAOG,eAAeC;EACpC;;;;EAKA,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKR,kBAAmB,MAAKA,oBAAoBS,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKX;EACd;;;;;EAMA,IAAcY,sBAA6D;AACzE,QAAI,CAAC,KAAKX,qBAAsB,MAAKA,uBAAuBY,iCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKV;EACd;;;;;EAMA,IAAcU,mBAAmE;AAC/E,WACE,KAAKR,QAAQQ,oBAAoB;MAC/B,CAAC,KAAKT,kBAAkB,GAAG;QACzB;UACEY,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;;IAEJ;EAEJ;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AACnF,UAAMC,UAAUD,SAASE,OAAOC,sBAAkD,KAAKnB,kBAAkB,CAAA;AACzG,QAAIiB,QAAQG,SAAS,GAAG;AACtB,aAAO,MAAMC,QAAQC,IACnBL,QAAQM,IAAI,OAAOC,UAAAA;AACjB,cAAMC,SAAS,MAAMC,eACnB;UAACF;WACD,KAAKd,qBACL,KAAKP,gBAAgB;AAGvBsB,eAAOE,UAAU;UAAC,KAAKvB;;AAEvB,eAAOqB,OAAOG;AAEd,eAAO,MAAM,IAAIC,eAAwB;UAAEC,QAAQ,KAAK3B;QAAiB,CAAA,EAAGsB,OAAOA,MAAAA,EAAQM,MAAK;MAClG,CAAA,CAAA;IAEJ;AACA,WAAO,CAAA;EACT;AACF;;;ACvHA,SAASC,mBAAmB;AAC5B,SAASC,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,sBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAChC,SAASC,oCAAAA,yCAAwC;AAMjD,SACEC,iEACAC,4CAAAA,iDACK;AAEP,SAASC,kBAAAA,uBAAsB;AAE/B,SAASC,oBAAoB;AAC7B,SAASC,2CAA2C;AAIpD,IAAMC,aAAa;AAOZ,IAAMC,sDAAN,cAEGC,iBAAAA;EAlCV,OAkCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKF,kBAAmB,MAAKA,oBAAoBG,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKL;EACd;;;;;EAMA,IAAcM,sBAA6D;AACzE,QAAI,CAAC,KAAKL,qBAAsB,MAAKA,uBAAuBM,kCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKJ;EACd;;;;;EAMA,IAAcI,mBAAmE;AAC/E,WAAOG,SAAS,KAAKC,QAAQJ,kBAAkB,MAAM,GAAGZ,UAAAA,2CAAqD;EAC/G;EAEA,MAAyBiB,cAAcC,WAAsB,CAAA,GAAwB;AAEnF,UAAMC,0BAA0BD,SAC7BE,OAAOC,cAAAA,EACPD,OAAOE,CAAAA,OAAMC,YAAYD,GAAGE,iBAAiB,KAAKf,gBAAgB,CAAA;AAErE,UAAMgB,oBAAoBP,SAASE,OAAOM,CAAAA,MAAK,KAAKC,mBAAmBD,CAAAA,CAAAA;AAEvE,QAAIP,wBAAwBS,WAAW,KAAKH,kBAAkBG,WAAW,EAAG,QAAO,CAAA;AAEnF,UAAM,CAACC,cAAcC,iBAAAA,IAAqB,MAAMC,QAAQC,IAAI;MAC1DC,gBAAeC,cAAcf,uBAAAA;MAC7Bc,gBAAeC,cAAcT,iBAAAA;KAC9B;AAGD,UAAMU,uBAA0C,CAAA;AAGhD,eAAW,CAACC,QAAQd,EAAAA,KAAOZ,OAAO2B,QAAQR,YAAAA,GAAyC;AAEjF,YAAMS,eAAeC,oCAAoCjB,IAAI,KAAKb,gBAAgB;AAGlF,iBAAW+B,eAAeF,cAAc;AACtC,cAAMb,qBAAoBe,YAAYC,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK,EAAEtB,OAAOuB,MAAAA;AAIlF,YAAIlB,mBAAkBG,WAAW,KAAKnB,iBAAiBmB,QAAQ;AAC7DO,+BAAqBS,KAAK;YAACR;eAAWI;WAAY;QACpD;MACF;IACF;AAGA,UAAMK,UAAU,MAAMd,QAAQC,IAC5BG,qBAAqBM,IAAiD,OAAO,CAACL,QAAWU,sBAAAA,MAAoB;AAC3G,YAAMC,iBAAiBD,oBAAoBL,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK;AAE9E,YAAMM,cAAcD,eAAeE,QAAQ,CAACC,YAAAA;AAE1C,cAAMC,eAAe,KAAKtC,oBAAoBqC,QAAQE,MAAM;AAE5D,eAAOD,eAAeA,aAAaV,IAAIY,CAAAA,cAAaA,UAAUH,OAAAA,CAAAA,IAAY,CAAA;MAC5E,CAAA;AAEA,YAAMI,UAAoB;QAAClB;WAAWU;;AAEtC,aAAO,MAAM,IAAIb,gBAAmD;QAAEmB,QAAQG;MAAyC,CAAA,EACpHC,OAAO9C,OAAO+C,OAAO;QAAEH;MAAQ,GAAA,GAAMN,WAAAA,CAAAA,EACrCU,MAAK;IACV,CAAA,CAAA;AAEF,WAAOb,QAAQc,KAAI;EACrB;;;;;;EAOUhC,qBAAqB,wBAACiC,MAAAA;AAC9B,WAAOC,aAAaD,CAAAA,KAAM,KAAKnD,iBAAiBqD,SAASF,GAAGR,MAAAA;EAC9D,GAF+B;;;;;;EASrBW,oBAAoB,wBAACX,WAAAA;AAC7B,WAAO,OAAOA,WAAW,WAAW,KAAK3C,iBAAiBqD,SAASV,MAAAA,IAAU;EAC/E,GAF8B;AAGhC;;;AChJA,SAASY,mBAAAA,wBAAuB;AAChC,SAASC,oCAAoC;AAC7C,SAASC,0FAA0F;AAO5F,IAAMC,yEAAN,cAAqFC,iBAAAA;EAT5F,OAS4FA;;;EAC1F,OAAyBC,gBAA0B;OAC9C,MAAMA;IACTC;;EAGF,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AAKnF,UAAMC,YAAYD,SAASE,OAAOC,CAAAA,MAAK,CAACC,6BAA6BD,CAAAA,CAAAA;AACrE,WAAO,MAAME,QAAQC,QAAQL,SAAAA;EAC/B;AACF;;;AC7BA,SAASM,YAAAA,iBAAgB;AACzB,SAASC,UAAAA,eAAc;AAEvB,SAASC,wBAAwB;AAEjC,SAASC,kBAAAA,uBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAGhC,SAASC,sCAAsC;AAG/C,SAASC,uEAAuE;AAChF,SAASC,sBAAsB;AAI/B,SAASC,eAAeC,yBAAyB;AACjD,SAASC,kBAAAA,uBAAsB;AAE/B,SAASC,uCAAAA,4CAA2C;AAEpD,SAASC,uBAAuB;AAwBhC,IAAMC,QAAQ;AAKd,IAAMC,cAAa;AAKZ,IAAMC,sDAAN,cAEGC,iBAAAA;EA1DV,OA0DUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;;;;EAKA,IAAcE,kBAA4B;AACxC,UAAMC,UAAU,KAAKF,OAAOG,QAAQF;AACpC,WAAO;MAACG;SAAqBF,WAAW,CAAA;;EAC1C;EAEA,MAAyBG,cAAcC,WAAsB,CAAA,GAAiD;AAE5G,UAAMC,YAAYD,SAASE,KAAKC,aAAAA;AAEhC,QAAI,CAACF,UAAW,QAAO;MAAC;QAAEG,QAAQC;QAAmBC,OAAO;UAAEC,QAAQ;QAAE;MAAE;;AAE1E,UAAM,EAAEA,OAAM,IAAKN,UAAUK;AAE7B,UAAME,sBAAsB,MAAM,KAAKC,+BAA8B;AACrE,QAAI,CAACD,oBAAqB,QAAO;MAACP;;AAClC,UAAMS,QAAQ,MAAM,IAAIC,gBAAgD;MAAEP,QAAQQ;IAA+B,CAAA,EAC9GC,OAAO;MACNC,OAAO,KAAKrB;MAAqBc;MAAQtB;MAAOU,iBAAiB,KAAKA;IACxE,CAAA,EACCoB,MAAK;AACR,UAAMC,QAAQ,MAAMR,oBAAoBS,OAAO;MAACP;KAAM;AACtD,QAAIM,MAAME,WAAW,EAAG,QAAO;MAACjB;;AAEhC,UAAMkB,kBAAkB,MAAM,KAAKC,qBAAoB;AACvD,QAAI,CAACD,gBAAiB,QAAO;MAAClB;;AAC9B,UAAMoB,MAAML,MAAMnB,OAAOyB,eAAAA;AACzB,UAAMC,mBAAqC,MAAMC,QAAQC,IAAIJ,IAAIK,IAAIC,CAAAA,OAAM,KAAKC,0BAA0BD,IAAIR,eAAAA,CAAAA,CAAAA,GAC3GtB,OAAOgC,OAAAA,EACPC,KAAI;AACP,UAAMC,YAAY;MAAE3B,QAAQC;MAAmBC,OAAO;QAAE,GAAGL,UAAUK;QAAOC,QAAQA,SAASS,MAAME;MAAO;IAAE;AAC5G,WAAO;MAACa;SAAcR;;EACxB;;;;;EAMA,MAAgBH,uBAA8D;AAE5E,UAAMY,OAAyBC,UAC7B,KAAKvC,QAAQwC,cAAcC,WAC3B,MAAM,GAAGjD,WAAAA,mDAA6D;AAGxE,UAAMkD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK,QAAOE;AAEjB,WAAOC,iBAAiBC,KAAKJ,KAAK,KAAKK,OAAO;EAChD;;;;;EAMA,MAAgBhC,iCAAiC;AAE/C,UAAMuB,OAAyBC,UAC7B,KAAKvC,QAAQwC,cAAc1B,qBAC3B,MAAM,GAAGtB,WAAAA,6DAAuE;AAGlF,UAAMkD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK;AAEV,WAAOM,eAAeF,KAMpBJ,KAAK,KAAKK,OAAO;EACrB;EAEA,MAAgBb,0BAA0BD,IAAkBQ,WAAqE;AAC/H,UAAMQ,eAAeC,qCAAoCjB,IAAI,KAAKhC,eAAe,EAAEmC,KAAI;AACvF,QAAIa,aAAazB,WAAW,EAAG,QAAOoB;AACtC,UAAMO,SAAS,IAAIC,IAAIH,YAAAA;AACvB,UAAMpB,kBAAkB,MAAMY,UAAUY,IAAI;SAAIF;KAAO;AACvD,WAAO;MAAClB;SAAOJ;;EACjB;AACF;","names":["IndexingDiviner","TemporalIndexingDivinerConfigSchema","TemporalIndexingDiviner","IndexingDiviner","configSchemas","TemporalIndexingDivinerConfigSchema","defaultConfigSchema","startHandler","AbstractDiviner","jsonPathToTransformersDictionary","reducePayloads","PayloadDivinerQuerySchema","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isPayloadOfSchemaType","TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","divinerQuerySchema","config","PayloadDivinerQuerySchema","indexQuerySchema","indexSchema","TemporalIndexingDivinerResultIndexSchema","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","defaultValue","destinationField","sourcePathExpression","divineHandler","payloads","queries","filter","isPayloadOfSchemaType","length","Promise","all","map","query","fields","reducePayloads","schemas","sources","PayloadBuilder","schema","build","containsAll","assertEx","exists","isBoundWitness","AbstractDiviner","jsonPathToTransformersDictionary","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isAnyPayload","intraBoundwitnessSchemaCombinations","moduleName","TemporalIndexingDivinerIndexCandidateToIndexDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","assertEx","config","divineHandler","payloads","indexableBoundWitnesses","filter","isBoundWitness","bw","containsAll","payload_schemas","indexablePayloads","p","isIndexablePayload","length","bwDictionary","payloadDictionary","Promise","all","PayloadBuilder","toDataHashMap","validIndexableTuples","bwHash","entries","combinations","intraBoundwitnessSchemaCombinations","combination","map","hash","exists","push","indexes","sourcePayloadHashes","sourcePayloads","indexFields","flatMap","payload","transformers","schema","transform","sources","TemporalIndexingDivinerResultIndexSchema","fields","assign","build","flat","x","isAnyPayload","includes","isIndexableSchema","AbstractDiviner","isPayloadDivinerQueryPayload","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","defaultConfigSchema","labels","divineHandler","payloads","responses","filter","p","isPayloadDivinerQueryPayload","Promise","resolve","assertEx","exists","ArchivistWrapper","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","DivinerWrapper","isModuleState","ModuleStateSchema","PayloadBuilder","intraBoundwitnessSchemaCombinations","TimestampSchema","order","moduleName","TemporalIndexingDivinerStateToIndexCandidateDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","defaultConfigSchema","labels","payloadDivinerLimit","config","payload_schemas","schemas","filter","TimestampSchema","divineHandler","payloads","lastState","find","isModuleState","schema","ModuleStateSchema","state","offset","boundWitnessDiviner","getBoundWitnessDivinerForStore","query","PayloadBuilder","BoundWitnessDivinerQuerySchema","fields","limit","build","batch","divine","length","sourceArchivist","getArchivistForStore","bws","isBoundWitness","indexCandidates","Promise","all","map","bw","getPayloadsInBoundWitness","exists","flat","nextState","name","assertEx","payloadStore","archivist","mod","resolve","undefined","ArchivistWrapper","wrap","account","DivinerWrapper","combinations","intraBoundwitnessSchemaCombinations","hashes","Set","get"]}
+{"version":3,"sources":["../../src/Diviner.ts","../../src/DivinerQueryToIndexQueryDiviner/Diviner.ts","../../src/IndexCandidateToIndexDiviner/Diviner.ts","../../src/IndexQueryResponseToDivinerQueryResponseDiviner/Diviner.ts","../../src/StateToIndexCandidateDiviner/Diviner.ts"],"sourcesContent":["import { IndexingDiviner } from '@xyo-network/diviner-indexing-memory'\nimport type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport type { TemporalIndexingDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\nexport class TemporalIndexingDiviner<\n  TParams extends TemporalIndexingDivinerParams = TemporalIndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends IndexingDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerConfigSchema\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    return true\n  }\n}\n","import type { Hash } from '@xylabs/hex'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary, reducePayloads } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport type { TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { isPayloadOfSchemaType } from '@xyo-network/payload-model'\n// TODO: Inherit from JsonPathAggregateDiviner\n/**\n * A diviner that converts diviner query to index query\n */\nexport class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<\n  TParams extends TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas = [...super.configSchemas, TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema]\n  static override readonly defaultConfigSchema = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'divinerQueryToIndexQueryDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * The schema of the diviner query payloads\n   */\n  protected get divinerQuerySchema(): string {\n    return this.config.divinerQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index query payloads\n   */\n  protected get indexQuerySchema(): string {\n    return this.config.indexQuerySchema ?? PayloadDivinerQuerySchema\n  }\n\n  /**\n   * The schema of the index payloads\n   */\n  protected get indexSchema(): string {\n    return this.config.indexSchema ?? TemporalIndexingDivinerResultIndexSchema\n  }\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return (\n      this.config?.schemaTransforms ?? {\n        [this.divinerQuerySchema]: [\n          {\n            defaultValue: 1,\n            destinationField: 'limit',\n            sourcePathExpression: '$.limit',\n          },\n          {\n            defaultValue: 0,\n            destinationField: 'offset',\n            sourcePathExpression: '$.offset',\n          },\n          {\n            defaultValue: 'desc',\n            destinationField: 'order',\n            sourcePathExpression: '$.order',\n          },\n        ],\n      }\n    )\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    const queries = payloads.filter(isPayloadOfSchemaType<PayloadDivinerQueryPayload>(this.divinerQuerySchema))\n    if (queries.length > 0) {\n      return await Promise.all(\n        queries.map(async (query) => {\n          const fields = await reducePayloads<PayloadDivinerQueryPayload & { sources?: Hash[] }>(\n            [query],\n            this.payloadTransformers,\n            this.indexQuerySchema,\n          )\n          // TODO: Make index schema configurable\n          fields.schemas = [this.indexSchema]\n          // TODO: Make sources not need to be deleted\n          delete fields.sources\n          // TODO: Add support for additional filters\n          return await new PayloadBuilder<Payload>({ schema: this.indexQuerySchema }).fields(fields).build()\n        }),\n      )\n    }\n    return []\n  }\n}\n","import { containsAll } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { jsonPathToTransformersDictionary } from '@xyo-network/diviner-jsonpath-aggregate-memory'\nimport type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'\nimport type {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n  TemporalIndexingDivinerResultIndex,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport {\n  TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema,\n  TemporalIndexingDivinerResultIndexSchema,\n} from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\nimport { isAnyPayload } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\n\ntype IndexableHashes = [Hash, ...Hash[]]\n\nconst moduleName = 'TemporalIndexingDivinerIndexCandidateToIndexDiviner'\n\n/**\n * Diviner which transforms index candidates to indexes using JSON Path to map\n * source properties in the supplied payloads to destination fields in the\n * resultant index\n */\nexport class TemporalIndexingDivinerIndexCandidateToIndexDiviner<\n  TParams extends TemporalIndexingDivinerIndexCandidateToIndexDivinerParams = TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,\n> extends AbstractDiviner<TParams> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexCandidateToIndexDiviner',\n  }\n\n  private _indexableSchemas: string[] | undefined\n  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined\n\n  /**\n   * List of indexable schemas for this diviner\n   */\n  protected get indexableSchemas(): string[] {\n    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms)\n    return this._indexableSchemas\n  }\n\n  /**\n   * Dictionary of schemas to payload transformers for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {\n    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)\n    return this._payloadTransformers\n  }\n\n  /**\n   * The dictionary of schemas to JSON Path transform expressions for creating indexes\n   * from the payloads within a Bound Witness\n   */\n  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {\n    return assertEx(this.config?.schemaTransforms, () => `${moduleName}: Missing config.schemaTransforms section`)\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // If the Bound Witness does not contain all the required schemas do not index it\n    const indexableBoundWitnesses = payloads\n      .filter(isBoundWitness)\n      .filter(bw => containsAll(bw.payload_schemas, this.indexableSchemas))\n    // If the Payload is not one of the indexable schemas do not index it\n    const indexablePayloads = payloads.filter(p => this.isIndexablePayload(p))\n    // If there is nothing to index, return an empty array\n    if (indexableBoundWitnesses.length === 0 || indexablePayloads.length === 0) return []\n    // Hash all the indexable data once\n    const [bwDictionary, payloadDictionary] = await Promise.all([\n      PayloadBuilder.toDataHashMap(indexableBoundWitnesses),\n      PayloadBuilder.toDataHashMap(indexablePayloads),\n    ])\n\n    // Initialize the array for validIndexableTuples outside of the loop\n    const validIndexableTuples: IndexableHashes[] = []\n\n    // Iterate over each entry in bwDictionary\n    for (const [bwHash, bw] of Object.entries(bwDictionary) as [Hash, BoundWitness][]) {\n      // Find the combinations of payloads that satisfy the required schemas\n      const combinations = intraBoundwitnessSchemaCombinations(bw, this.indexableSchemas)\n\n      // Iterate over each combination\n      for (const combination of combinations) {\n        const indexablePayloads = combination.map(hash => payloadDictionary[hash]).filter(exists)\n\n        // If we found the right amount of indexable payloads (of the correct schema as checked\n        // above) in this BW, then index it\n        if (indexablePayloads.length === this.indexableSchemas.length) {\n          validIndexableTuples.push([bwHash, ...combination])\n        }\n      }\n    }\n\n    // Create the indexes from the tuples\n    const indexes = await Promise.all(\n      validIndexableTuples.map<Promise<TemporalIndexingDivinerResultIndex>>(async ([bwHash, ...sourcePayloadHashes]) => {\n        const sourcePayloads = sourcePayloadHashes.map(hash => payloadDictionary[hash])\n        // Use the payload transformers to convert the fields from the source payloads to the destination fields\n        const indexFields = sourcePayloads.flatMap((payload) => {\n          // Find the transformers for this payload\n          const transformers = this.payloadTransformers[payload.schema]\n          // If transformers exist, apply them to the payload otherwise return an empty array\n          return transformers ? transformers.map(transform => transform(payload)) : []\n        })\n        // Include all the sources for reference\n        const sources: string[] = [bwHash, ...sourcePayloadHashes]\n        // Build and return the index\n        return await new PayloadBuilder<TemporalIndexingDivinerResultIndex>({ schema: TemporalIndexingDivinerResultIndexSchema })\n          .fields(Object.assign({ sources }, ...indexFields))\n          .build()\n      }),\n    )\n    return indexes.flat()\n  }\n\n  /**\n   * Identifies if a payload is one that is indexed by this diviner\n   * @param x The candidate payload\n   * @returns True if the payload is one indexed by this diviner, false otherwise\n   */\n  protected isIndexablePayload = (x: unknown) => {\n    return isAnyPayload(x) && this.indexableSchemas.includes(x?.schema)\n  }\n\n  /**\n   * Identifies if a schema is one that is indexed by this diviner\n   * @param schema The candidate schema\n   * @returns True if this schema is one indexed by this diviner, false otherwise\n   */\n  protected isIndexableSchema = (schema?: string | null) => {\n    return typeof schema === 'string' ? this.indexableSchemas.includes(schema) : false\n  }\n}\n","import { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport type { Labels } from '@xyo-network/module-model'\nimport type { Payload, Schema } from '@xyo-network/payload-model'\n\n/**\n * Transforms an ImageThumbnailIndex response into an ImageThumbnailResponse response\n */\nexport class TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner extends AbstractDiviner {\n  static override readonly configSchemas: Schema[] = [\n    ...super.configSchemas,\n    TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema,\n  ]\n\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'indexQueryResponseToDivinerQueryResponseDiviner',\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {\n    // NOTE: We're not doing anything with the query payloads but some diviners\n    // might want to use this to transform from the query to the response (for example\n    // if we use a plaintext value in the query to generate a hash key in the index)\n    // const queries = payloads.filter(isPayloadDivinerQueryPayload)\n    const responses = payloads.filter(p => !isPayloadDivinerQueryPayload(p))\n    return await Promise.resolve(responses)\n  }\n}\n","import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hex } from '@xylabs/hex'\nimport type { ArchivistInstance } from '@xyo-network/archivist-model'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport type { BoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-abstract'\nimport type { BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'\nimport { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'\nimport type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'\nimport { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport type {\n  Labels, ModuleIdentifier, ModuleState,\n} from '@xyo-network/module-model'\nimport { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema,\n  WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport { StorageMetaConstants } from '@xyo-network/payload-model'\nimport { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'\nimport type { TimeStamp } from '@xyo-network/witness-timestamp'\nimport { TimestampSchema } from '@xyo-network/witness-timestamp'\n\n/**\n * All Payload types involved in index candidates for indexing\n */\nexport type IndexCandidate = BoundWitness | Payload | TimeStamp\n\n/**\n * The response from the TemporalStateToIndexCandidateDiviner\n */\nexport type TemporalStateToIndexCandidateDivinerResponse = [\n  /**\n   * The next state of the diviner\n   */\n  nextState: ModuleState<IndexingDivinerState>,\n  /**\n   * The index candidates\n   */\n  ...IndexCandidate[],\n]\n\n/**\n * The default order to search Bound Witnesses to identify index candidates\n */\nconst order = 'asc'\n\n/**\n * The name of the module (for logging purposes)\n */\nconst moduleName = 'TemporalIndexingDivinerStateToIndexCandidateDiviner'\n\n/**\n * Transforms candidates for image thumbnail indexing into their indexed representation\n */\nexport class TemporalIndexingDivinerStateToIndexCandidateDiviner<\n  TParams extends TemporalIndexingDivinerStateToIndexCandidateDivinerParams = TemporalIndexingDivinerStateToIndexCandidateDivinerParams,\n> extends AbstractDiviner<TParams, Payload, ModuleState | IndexCandidate> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema\n  static override labels: Labels = {\n    ...super.labels,\n    'network.xyo.diviner.stage': 'stateToIndexCandidateDiviner',\n  }\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1000\n  }\n\n  /**\n   * The required payload_schemas within BoundWitnesses to identify index candidates\n   */\n  protected get payload_schemas(): string[] {\n    const schemas = this.config.filter?.payload_schemas\n    return [TimestampSchema, ...(schemas ?? [])]\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<[ModuleState, ...IndexCandidate[]]> {\n    // Retrieve the last state from what was passed in\n    const lastState = payloads.find(isModuleState<IndexingDivinerState>)\n    // If there is no last state, start from the beginning\n    if (!lastState) return [{ schema: ModuleStateSchema, state: { cursor: StorageMetaConstants.minLocalSequence } }]\n    // Otherwise, get the last offset\n    const { cursor } = lastState.state\n    // Get next batch of results starting from the offset\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore()\n    if (!boundWitnessDiviner) return [lastState]\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit, cursor: cursor as Hex, order, payload_schemas: this.payload_schemas,\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return [lastState]\n    // Get source data\n    const sourceArchivist = await this.getArchivistForStore()\n    if (!sourceArchivist) return [lastState]\n    const bws = batch.filter(isBoundWitness)\n    const indexCandidates: IndexCandidate[] = (await Promise.all(bws.map(bw => this.getPayloadsInBoundWitness(bw, sourceArchivist))))\n      .filter(exists)\n      .flat()\n    const nextState = { schema: ModuleStateSchema, state: { ...lastState.state, cursor: batch.at(-1)?._sequence } }\n    return [nextState, ...indexCandidates]\n  }\n\n  /**\n   * Retrieves the archivist for the payloadStore\n   * @returns The archivist for the payloadStore or undefined if not resolvable\n   */\n  protected async getArchivistForStore(): Promise<ArchivistWrapper | undefined> {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.archivist,\n      () => `${moduleName}: Config for payloadStore.archivist not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return undefined\n    // Return the wrapped archivist\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the payloadStore\n   * @returns The BoundWitness Diviner for the payloadStore or undefined if not resolvable\n   */\n  protected async getBoundWitnessDivinerForStore() {\n    // It should be defined, so we'll error if it's not\n    const name: ModuleIdentifier = assertEx(\n      this.config?.payloadStore?.boundWitnessDiviner,\n      () => `${moduleName}: Config for payloadStore.boundWitnessDiviner not specified`,\n    )\n    // It might not be resolvable (yet), so we'll return undefined if it's not\n    const mod = await this.resolve(name)\n    if (!mod) return\n    // Return the wrapped diviner\n    return DivinerWrapper.wrap<\n      DivinerWrapper<\n        BoundWitnessDiviner<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, BoundWitness>,\n        BoundWitnessDivinerQueryPayload,\n        WithStorageMeta<BoundWitness>\n      >\n    >(mod, this.account)\n  }\n\n  protected async getPayloadsInBoundWitness(bw: BoundWitness, archivist: ArchivistInstance): Promise<IndexCandidate[] | undefined> {\n    const combinations = intraBoundwitnessSchemaCombinations(bw, this.payload_schemas).flat()\n    if (combinations.length === 0) return undefined\n    const hashes = new Set(combinations)\n    const indexCandidates = await archivist.get([...hashes])\n    return [bw, ...indexCandidates]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,uBAAuB;AAGhC,SAASC,2CAA2C;AAG7C,IAAMC,0BAAN,cASGC,gBAAAA;EAfV,OAeUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAEvD,MAAyBE,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,WAAO;EACT;AACF;;;ACtBA,SAASC,uBAAuB;AAChC,SAASC,kCAAkCC,sBAAsB;AAGjE,SAASC,iCAAiC;AAE1C,SACEC,oEACAC,gDACK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,6BAA6B;AAK/B,IAAMC,yDAAN,cAEGC,gBAAAA;EApBV,OAoBUA;;;EACR,OAAyBC,gBAAgB;OAAI,MAAMA;IAAeC;;EAClE,OAAyBC,sBAAsBD;EAC/C,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,qBAA6B;AACzC,WAAO,KAAKC,OAAOD,sBAAsBE;EAC3C;;;;EAKA,IAAcC,mBAA2B;AACvC,WAAO,KAAKF,OAAOE,oBAAoBD;EACzC;;;;EAKA,IAAcE,cAAsB;AAClC,WAAO,KAAKH,OAAOG,eAAeC;EACpC;;;;EAKA,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKR,kBAAmB,MAAKA,oBAAoBS,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKX;EACd;;;;;EAMA,IAAcY,sBAA6D;AACzE,QAAI,CAAC,KAAKX,qBAAsB,MAAKA,uBAAuBY,iCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKV;EACd;;;;;EAMA,IAAcU,mBAAmE;AAC/E,WACE,KAAKR,QAAQQ,oBAAoB;MAC/B,CAAC,KAAKT,kBAAkB,GAAG;QACzB;UACEY,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;QACA;UACEF,cAAc;UACdC,kBAAkB;UAClBC,sBAAsB;QACxB;;IAEJ;EAEJ;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AACnF,UAAMC,UAAUD,SAASE,OAAOC,sBAAkD,KAAKnB,kBAAkB,CAAA;AACzG,QAAIiB,QAAQG,SAAS,GAAG;AACtB,aAAO,MAAMC,QAAQC,IACnBL,QAAQM,IAAI,OAAOC,UAAAA;AACjB,cAAMC,SAAS,MAAMC,eACnB;UAACF;WACD,KAAKd,qBACL,KAAKP,gBAAgB;AAGvBsB,eAAOE,UAAU;UAAC,KAAKvB;;AAEvB,eAAOqB,OAAOG;AAEd,eAAO,MAAM,IAAIC,eAAwB;UAAEC,QAAQ,KAAK3B;QAAiB,CAAA,EAAGsB,OAAOA,MAAAA,EAAQM,MAAK;MAClG,CAAA,CAAA;IAEJ;AACA,WAAO,CAAA;EACT;AACF;;;ACvHA,SAASC,mBAAmB;AAC5B,SAASC,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,sBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAChC,SAASC,oCAAAA,yCAAwC;AAMjD,SACEC,iEACAC,4CAAAA,iDACK;AAEP,SAASC,kBAAAA,uBAAsB;AAE/B,SAASC,oBAAoB;AAC7B,SAASC,2CAA2C;AAIpD,IAAMC,aAAa;AAOZ,IAAMC,sDAAN,cAEGC,iBAAAA;EAlCV,OAkCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEQC;EACAC;;;;EAKR,IAAcC,mBAA6B;AACzC,QAAI,CAAC,KAAKF,kBAAmB,MAAKA,oBAAoBG,OAAOC,KAAK,KAAKC,gBAAgB;AACvF,WAAO,KAAKL;EACd;;;;;EAMA,IAAcM,sBAA6D;AACzE,QAAI,CAAC,KAAKL,qBAAsB,MAAKA,uBAAuBM,kCAAiC,KAAKF,gBAAgB;AAClH,WAAO,KAAKJ;EACd;;;;;EAMA,IAAcI,mBAAmE;AAC/E,WAAOG,SAAS,KAAKC,QAAQJ,kBAAkB,MAAM,GAAGZ,UAAAA,2CAAqD;EAC/G;EAEA,MAAyBiB,cAAcC,WAAsB,CAAA,GAAwB;AAEnF,UAAMC,0BAA0BD,SAC7BE,OAAOC,cAAAA,EACPD,OAAOE,CAAAA,OAAMC,YAAYD,GAAGE,iBAAiB,KAAKf,gBAAgB,CAAA;AAErE,UAAMgB,oBAAoBP,SAASE,OAAOM,CAAAA,MAAK,KAAKC,mBAAmBD,CAAAA,CAAAA;AAEvE,QAAIP,wBAAwBS,WAAW,KAAKH,kBAAkBG,WAAW,EAAG,QAAO,CAAA;AAEnF,UAAM,CAACC,cAAcC,iBAAAA,IAAqB,MAAMC,QAAQC,IAAI;MAC1DC,gBAAeC,cAAcf,uBAAAA;MAC7Bc,gBAAeC,cAAcT,iBAAAA;KAC9B;AAGD,UAAMU,uBAA0C,CAAA;AAGhD,eAAW,CAACC,QAAQd,EAAAA,KAAOZ,OAAO2B,QAAQR,YAAAA,GAAyC;AAEjF,YAAMS,eAAeC,oCAAoCjB,IAAI,KAAKb,gBAAgB;AAGlF,iBAAW+B,eAAeF,cAAc;AACtC,cAAMb,qBAAoBe,YAAYC,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK,EAAEtB,OAAOuB,MAAAA;AAIlF,YAAIlB,mBAAkBG,WAAW,KAAKnB,iBAAiBmB,QAAQ;AAC7DO,+BAAqBS,KAAK;YAACR;eAAWI;WAAY;QACpD;MACF;IACF;AAGA,UAAMK,UAAU,MAAMd,QAAQC,IAC5BG,qBAAqBM,IAAiD,OAAO,CAACL,QAAWU,sBAAAA,MAAoB;AAC3G,YAAMC,iBAAiBD,oBAAoBL,IAAIC,CAAAA,SAAQZ,kBAAkBY,IAAAA,CAAK;AAE9E,YAAMM,cAAcD,eAAeE,QAAQ,CAACC,YAAAA;AAE1C,cAAMC,eAAe,KAAKtC,oBAAoBqC,QAAQE,MAAM;AAE5D,eAAOD,eAAeA,aAAaV,IAAIY,CAAAA,cAAaA,UAAUH,OAAAA,CAAAA,IAAY,CAAA;MAC5E,CAAA;AAEA,YAAMI,UAAoB;QAAClB;WAAWU;;AAEtC,aAAO,MAAM,IAAIb,gBAAmD;QAAEmB,QAAQG;MAAyC,CAAA,EACpHC,OAAO9C,OAAO+C,OAAO;QAAEH;MAAQ,GAAA,GAAMN,WAAAA,CAAAA,EACrCU,MAAK;IACV,CAAA,CAAA;AAEF,WAAOb,QAAQc,KAAI;EACrB;;;;;;EAOUhC,qBAAqB,wBAACiC,MAAAA;AAC9B,WAAOC,aAAaD,CAAAA,KAAM,KAAKnD,iBAAiBqD,SAASF,GAAGR,MAAAA;EAC9D,GAF+B;;;;;;EASrBW,oBAAoB,wBAACX,WAAAA;AAC7B,WAAO,OAAOA,WAAW,WAAW,KAAK3C,iBAAiBqD,SAASV,MAAAA,IAAU;EAC/E,GAF8B;AAGhC;;;AChJA,SAASY,mBAAAA,wBAAuB;AAChC,SAASC,oCAAoC;AAC7C,SAASC,0FAA0F;AAO5F,IAAMC,yEAAN,cAAqFC,iBAAAA;EAT5F,OAS4FA;;;EAC1F,OAAyBC,gBAA0B;OAC9C,MAAMA;IACTC;;EAGF,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,MAAyBC,cAAcC,WAAsB,CAAA,GAAwB;AAKnF,UAAMC,YAAYD,SAASE,OAAOC,CAAAA,MAAK,CAACC,6BAA6BD,CAAAA,CAAAA;AACrE,WAAO,MAAME,QAAQC,QAAQL,SAAAA;EAC/B;AACF;;;AC7BA,SAASM,YAAAA,iBAAgB;AACzB,SAASC,UAAAA,eAAc;AAGvB,SAASC,wBAAwB;AAEjC,SAASC,kBAAAA,uBAAsB;AAC/B,SAASC,mBAAAA,wBAAuB;AAGhC,SAASC,sCAAsC;AAG/C,SAASC,uEAAuE;AAChF,SAASC,sBAAsB;AAI/B,SAASC,eAAeC,yBAAyB;AACjD,SAASC,kBAAAA,uBAAsB;AAK/B,SAASC,4BAA4B;AACrC,SAASC,uCAAAA,4CAA2C;AAEpD,SAASC,uBAAuB;AAwBhC,IAAMC,QAAQ;AAKd,IAAMC,cAAa;AAKZ,IAAMC,sDAAN,cAEGC,iBAAAA;EA/DV,OA+DUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,SAAiB;IAC/B,GAAG,MAAMA;IACT,6BAA6B;EAC/B;EAEA,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;;;;EAKA,IAAcE,kBAA4B;AACxC,UAAMC,UAAU,KAAKF,OAAOG,QAAQF;AACpC,WAAO;MAACG;SAAqBF,WAAW,CAAA;;EAC1C;EAEA,MAAyBG,cAAcC,WAAsB,CAAA,GAAiD;AAE5G,UAAMC,YAAYD,SAASE,KAAKC,aAAAA;AAEhC,QAAI,CAACF,UAAW,QAAO;MAAC;QAAEG,QAAQC;QAAmBC,OAAO;UAAEC,QAAQC,qBAAqBC;QAAiB;MAAE;;AAE9G,UAAM,EAAEF,OAAM,IAAKN,UAAUK;AAE7B,UAAMI,sBAAsB,MAAM,KAAKC,+BAA8B;AACrE,QAAI,CAACD,oBAAqB,QAAO;MAACT;;AAClC,UAAMW,QAAQ,IAAIC,gBAAgD;MAAET,QAAQU;IAA+B,CAAA,EACxGC,OAAO;MACNC,OAAO,KAAKvB;MAAqBc;MAAuBtB;MAAOU,iBAAiB,KAAKA;IACvF,CAAA,EACCsB,MAAK;AACR,UAAMC,QAAQ,MAAMR,oBAAoBS,OAAO;MAACP;KAAM;AACtD,QAAIM,MAAME,WAAW,EAAG,QAAO;MAACnB;;AAEhC,UAAMoB,kBAAkB,MAAM,KAAKC,qBAAoB;AACvD,QAAI,CAACD,gBAAiB,QAAO;MAACpB;;AAC9B,UAAMsB,MAAML,MAAMrB,OAAO2B,eAAAA;AACzB,UAAMC,mBAAqC,MAAMC,QAAQC,IAAIJ,IAAIK,IAAIC,CAAAA,OAAM,KAAKC,0BAA0BD,IAAIR,eAAAA,CAAAA,CAAAA,GAC3GxB,OAAOkC,OAAAA,EACPC,KAAI;AACP,UAAMC,YAAY;MAAE7B,QAAQC;MAAmBC,OAAO;QAAE,GAAGL,UAAUK;QAAOC,QAAQW,MAAMgB,GAAG,EAAC,GAAIC;MAAU;IAAE;AAC9G,WAAO;MAACF;SAAcR;;EACxB;;;;;EAMA,MAAgBH,uBAA8D;AAE5E,UAAMc,OAAyBC,UAC7B,KAAK3C,QAAQ4C,cAAcC,WAC3B,MAAM,GAAGrD,WAAAA,mDAA6D;AAGxE,UAAMsD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK,QAAOE;AAEjB,WAAOC,iBAAiBC,KAAKJ,KAAK,KAAKK,OAAO;EAChD;;;;;EAMA,MAAgBlC,iCAAiC;AAE/C,UAAMyB,OAAyBC,UAC7B,KAAK3C,QAAQ4C,cAAc5B,qBAC3B,MAAM,GAAGxB,WAAAA,6DAAuE;AAGlF,UAAMsD,MAAM,MAAM,KAAKC,QAAQL,IAAAA;AAC/B,QAAI,CAACI,IAAK;AAEV,WAAOM,eAAeF,KAMpBJ,KAAK,KAAKK,OAAO;EACrB;EAEA,MAAgBf,0BAA0BD,IAAkBU,WAAqE;AAC/H,UAAMQ,eAAeC,qCAAoCnB,IAAI,KAAKlC,eAAe,EAAEqC,KAAI;AACvF,QAAIe,aAAa3B,WAAW,EAAG,QAAOsB;AACtC,UAAMO,SAAS,IAAIC,IAAIH,YAAAA;AACvB,UAAMtB,kBAAkB,MAAMc,UAAUY,IAAI;SAAIF;KAAO;AACvD,WAAO;MAACpB;SAAOJ;;EACjB;AACF;","names":["IndexingDiviner","TemporalIndexingDivinerConfigSchema","TemporalIndexingDiviner","IndexingDiviner","configSchemas","TemporalIndexingDivinerConfigSchema","defaultConfigSchema","startHandler","AbstractDiviner","jsonPathToTransformersDictionary","reducePayloads","PayloadDivinerQuerySchema","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isPayloadOfSchemaType","TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","divinerQuerySchema","config","PayloadDivinerQuerySchema","indexQuerySchema","indexSchema","TemporalIndexingDivinerResultIndexSchema","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","defaultValue","destinationField","sourcePathExpression","divineHandler","payloads","queries","filter","isPayloadOfSchemaType","length","Promise","all","map","query","fields","reducePayloads","schemas","sources","PayloadBuilder","schema","build","containsAll","assertEx","exists","isBoundWitness","AbstractDiviner","jsonPathToTransformersDictionary","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","TemporalIndexingDivinerResultIndexSchema","PayloadBuilder","isAnyPayload","intraBoundwitnessSchemaCombinations","moduleName","TemporalIndexingDivinerIndexCandidateToIndexDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema","defaultConfigSchema","labels","_indexableSchemas","_payloadTransformers","indexableSchemas","Object","keys","schemaTransforms","payloadTransformers","jsonPathToTransformersDictionary","assertEx","config","divineHandler","payloads","indexableBoundWitnesses","filter","isBoundWitness","bw","containsAll","payload_schemas","indexablePayloads","p","isIndexablePayload","length","bwDictionary","payloadDictionary","Promise","all","PayloadBuilder","toDataHashMap","validIndexableTuples","bwHash","entries","combinations","intraBoundwitnessSchemaCombinations","combination","map","hash","exists","push","indexes","sourcePayloadHashes","sourcePayloads","indexFields","flatMap","payload","transformers","schema","transform","sources","TemporalIndexingDivinerResultIndexSchema","fields","assign","build","flat","x","isAnyPayload","includes","isIndexableSchema","AbstractDiviner","isPayloadDivinerQueryPayload","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema","defaultConfigSchema","labels","divineHandler","payloads","responses","filter","p","isPayloadDivinerQueryPayload","Promise","resolve","assertEx","exists","ArchivistWrapper","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","DivinerWrapper","isModuleState","ModuleStateSchema","PayloadBuilder","StorageMetaConstants","intraBoundwitnessSchemaCombinations","TimestampSchema","order","moduleName","TemporalIndexingDivinerStateToIndexCandidateDiviner","AbstractDiviner","configSchemas","TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema","defaultConfigSchema","labels","payloadDivinerLimit","config","payload_schemas","schemas","filter","TimestampSchema","divineHandler","payloads","lastState","find","isModuleState","schema","ModuleStateSchema","state","cursor","StorageMetaConstants","minLocalSequence","boundWitnessDiviner","getBoundWitnessDivinerForStore","query","PayloadBuilder","BoundWitnessDivinerQuerySchema","fields","limit","build","batch","divine","length","sourceArchivist","getArchivistForStore","bws","isBoundWitness","indexCandidates","Promise","all","map","bw","getPayloadsInBoundWitness","exists","flat","nextState","at","_sequence","name","assertEx","payloadStore","archivist","mod","resolve","undefined","ArchivistWrapper","wrap","account","DivinerWrapper","combinations","intraBoundwitnessSchemaCombinations","hashes","Set","get"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/diviner-temporal-indexing-memory",
-  "version": "3.6.0-rc.3",
+  "version": "3.6.0-rc.4",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -29,41 +29,41 @@
   "module": "dist/neutral/index.mjs",
   "types": "dist/neutral/index.d.ts",
   "dependencies": {
-    "@xylabs/array": "^4.4.12",
-    "@xylabs/assert": "^4.4.12",
-    "@xylabs/exists": "^4.4.12",
-    "@xylabs/hex": "^4.4.12",
-    "@xyo-network/archivist-model": "^3.6.0-rc.3",
-    "@xyo-network/archivist-wrapper": "^3.6.0-rc.3",
-    "@xyo-network/boundwitness-model": "^3.6.0-rc.3",
-    "@xyo-network/diviner-abstract": "^3.6.0-rc.3",
-    "@xyo-network/diviner-boundwitness-abstract": "^3.6.0-rc.3",
-    "@xyo-network/diviner-boundwitness-model": "^3.6.0-rc.3",
-    "@xyo-network/diviner-indexing-memory": "^3.6.0-rc.3",
-    "@xyo-network/diviner-indexing-model": "^3.6.0-rc.3",
-    "@xyo-network/diviner-jsonpath-aggregate-memory": "^3.6.0-rc.3",
-    "@xyo-network/diviner-jsonpath-model": "^3.6.0-rc.3",
-    "@xyo-network/diviner-model": "^3.6.0-rc.3",
-    "@xyo-network/diviner-payload-model": "^3.6.0-rc.3",
-    "@xyo-network/diviner-temporal-indexing-model": "^3.6.0-rc.3",
-    "@xyo-network/diviner-wrapper": "^3.6.0-rc.3",
-    "@xyo-network/module-model": "^3.6.0-rc.3",
-    "@xyo-network/payload-builder": "^3.6.0-rc.3",
-    "@xyo-network/payload-model": "^3.6.0-rc.3",
-    "@xyo-network/payload-utils": "^3.6.0-rc.3",
-    "@xyo-network/witness-timestamp": "^3.6.0-rc.3"
+    "@xylabs/array": "^4.4.13",
+    "@xylabs/assert": "^4.4.13",
+    "@xylabs/exists": "^4.4.13",
+    "@xylabs/hex": "^4.4.13",
+    "@xyo-network/archivist-model": "^3.6.0-rc.4",
+    "@xyo-network/archivist-wrapper": "^3.6.0-rc.4",
+    "@xyo-network/boundwitness-model": "^3.6.0-rc.4",
+    "@xyo-network/diviner-abstract": "^3.6.0-rc.4",
+    "@xyo-network/diviner-boundwitness-abstract": "^3.6.0-rc.4",
+    "@xyo-network/diviner-boundwitness-model": "^3.6.0-rc.4",
+    "@xyo-network/diviner-indexing-memory": "^3.6.0-rc.4",
+    "@xyo-network/diviner-indexing-model": "^3.6.0-rc.4",
+    "@xyo-network/diviner-jsonpath-aggregate-memory": "^3.6.0-rc.4",
+    "@xyo-network/diviner-jsonpath-model": "^3.6.0-rc.4",
+    "@xyo-network/diviner-model": "^3.6.0-rc.4",
+    "@xyo-network/diviner-payload-model": "^3.6.0-rc.4",
+    "@xyo-network/diviner-temporal-indexing-model": "^3.6.0-rc.4",
+    "@xyo-network/diviner-wrapper": "^3.6.0-rc.4",
+    "@xyo-network/module-model": "^3.6.0-rc.4",
+    "@xyo-network/payload-builder": "^3.6.0-rc.4",
+    "@xyo-network/payload-model": "^3.6.0-rc.4",
+    "@xyo-network/payload-utils": "^3.6.0-rc.4",
+    "@xyo-network/witness-timestamp": "^3.6.0-rc.4"
   },
   "devDependencies": {
-    "@xylabs/delay": "^4.4.12",
+    "@xylabs/delay": "^4.4.13",
     "@xylabs/ts-scripts-yarn3": "^4.2.4",
     "@xylabs/tsconfig": "^4.2.4",
-    "@xylabs/vitest-extended": "^4.4.12",
-    "@xyo-network/account": "^3.6.0-rc.3",
-    "@xyo-network/archivist-memory": "^3.6.0-rc.3",
-    "@xyo-network/boundwitness-builder": "^3.6.0-rc.3",
-    "@xyo-network/manifest": "^3.6.0-rc.3",
-    "@xyo-network/module-factory-locator": "^3.6.0-rc.3",
-    "@xyo-network/node-memory": "^3.6.0-rc.3",
+    "@xylabs/vitest-extended": "^4.4.13",
+    "@xyo-network/account": "^3.6.0-rc.4",
+    "@xyo-network/archivist-memory": "^3.6.0-rc.4",
+    "@xyo-network/boundwitness-builder": "^3.6.0-rc.4",
+    "@xyo-network/manifest": "^3.6.0-rc.4",
+    "@xyo-network/module-factory-locator": "^3.6.0-rc.4",
+    "@xyo-network/node-memory": "^3.6.0-rc.4",
     "typescript": "^5.7.2",
     "vitest": "^2.1.8"
   },

package/src/StateToIndexCandidateDiviner/Diviner.ts

@@ -1,5 +1,6 @@
 import { assertEx } from '@xylabs/assert'
 import { exists } from '@xylabs/exists'
+import type { Hex } from '@xylabs/hex'
 import type { ArchivistInstance } from '@xyo-network/archivist-model'
 import { ArchivistWrapper } from '@xyo-network/archivist-wrapper'
 import type { BoundWitness } from '@xyo-network/boundwitness-model'
@@ -17,7 +18,11 @@ import type {
 } from '@xyo-network/module-model'
 import { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
-import type { Payload, Schema } from '@xyo-network/payload-model'
+import type {
+  Payload, Schema,
+  WithStorageMeta,
+} from '@xyo-network/payload-model'
+import { StorageMetaConstants } from '@xyo-network/payload-model'
 import { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'
 import type { TimeStamp } from '@xyo-network/witness-timestamp'
 import { TimestampSchema } from '@xyo-network/witness-timestamp'
@@ -80,15 +85,15 @@ export class TemporalIndexingDivinerStateToIndexCandidateDiviner<
     // Retrieve the last state from what was passed in
     const lastState = payloads.find(isModuleState<IndexingDivinerState>)
     // If there is no last state, start from the beginning
-    if (!lastState) return [{ schema: ModuleStateSchema, state: { offset: 0 } }]
+    if (!lastState) return [{ schema: ModuleStateSchema, state: { cursor: StorageMetaConstants.minLocalSequence } }]
     // Otherwise, get the last offset
-    const { offset } = lastState.state
+    const { cursor } = lastState.state
     // Get next batch of results starting from the offset
     const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore()
     if (!boundWitnessDiviner) return [lastState]
-    const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
+    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
       .fields({
-        limit: this.payloadDivinerLimit, offset, order, payload_schemas: this.payload_schemas,
+        limit: this.payloadDivinerLimit, cursor: cursor as Hex, order, payload_schemas: this.payload_schemas,
       })
       .build()
     const batch = await boundWitnessDiviner.divine([query])
@@ -100,7 +105,7 @@ export class TemporalIndexingDivinerStateToIndexCandidateDiviner<
     const indexCandidates: IndexCandidate[] = (await Promise.all(bws.map(bw => this.getPayloadsInBoundWitness(bw, sourceArchivist))))
       .filter(exists)
       .flat()
-    const nextState = { schema: ModuleStateSchema, state: { ...lastState.state, offset: offset + batch.length } }
+    const nextState = { schema: ModuleStateSchema, state: { ...lastState.state, cursor: batch.at(-1)?._sequence } }
     return [nextState, ...indexCandidates]
   }
 
@@ -139,7 +144,7 @@ export class TemporalIndexingDivinerStateToIndexCandidateDiviner<
       DivinerWrapper<
         BoundWitnessDiviner<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, BoundWitness>,
         BoundWitnessDivinerQueryPayload,
-        BoundWitness
+        WithStorageMeta<BoundWitness>
       >
     >(mod, this.account)
   }