@xyo-network/diviner-image-thumbnail 2.77.4 → 2.77.5

package/dist/browser/Diviner/Diviner.d.cts

@@ -14,6 +14,8 @@ export declare class ImageThumbnailDiviner<TParams extends ImageThumbnailDiviner
     private _pollId?;
     get payloadDivinerLimit(): number;
     get pollFrequency(): number;
+    private static indexableData;
+    private static indexableHashes;
     /**
      * Works in the background to populate index for the Diviner
      * @returns

package/dist/browser/Diviner/Diviner.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAiB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAoFnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAmB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;mBAEoB,aAAa;mBA+Bb,eAAe;IAsBpC;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAwCnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}

package/dist/browser/Diviner/Diviner.d.mts

@@ -14,6 +14,8 @@ export declare class ImageThumbnailDiviner<TParams extends ImageThumbnailDiviner
     private _pollId?;
     get payloadDivinerLimit(): number;
     get pollFrequency(): number;
+    private static indexableData;
+    private static indexableHashes;
     /**
      * Works in the background to populate index for the Diviner
      * @returns

package/dist/browser/Diviner/Diviner.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAiB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAoFnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAmB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;mBAEoB,aAAa;mBA+Bb,eAAe;IAsBpC;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAwCnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}

package/dist/browser/Diviner/Diviner.d.ts

@@ -14,6 +14,8 @@ export declare class ImageThumbnailDiviner<TParams extends ImageThumbnailDiviner
     private _pollId?;
     get payloadDivinerLimit(): number;
     get pollFrequency(): number;
+    private static indexableData;
+    private static indexableHashes;
     /**
      * Works in the background to populate index for the Diviner
      * @returns

package/dist/browser/Diviner/Diviner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAiB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAoFnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAmB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;mBAEoB,aAAa;mBA+Bb,eAAe;IAsBpC;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAwCnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}

package/dist/browser/index.cjs

@@ -41,7 +41,7 @@ var import_module_model = require("@xyo-network/module-model");
 var import_payload_builder = require("@xyo-network/payload-builder");
 var import_witness_timestamp = require("@xyo-network/witness-timestamp");
 var moduleName = "ImageThumbnailDiviner";
-var ImageThumbnailDiviner = class extends import_abstract_diviner.AbstractDiviner {
+var ImageThumbnailDiviner = class _ImageThumbnailDiviner extends import_abstract_diviner.AbstractDiviner {
   static configSchemas = [import_image_thumbnail_payload_plugin.ImageThumbnailDivinerConfigSchema, import_diviner_model.DivinerConfigSchema];
   _pollId;
   get payloadDivinerLimit() {
@@ -50,6 +50,40 @@ var ImageThumbnailDiviner = class extends import_abstract_diviner.AbstractDivine
   get pollFrequency() {
     return this.config.pollFrequency ?? 1e4;
   }
+  static async indexableData(indexableHashes, archivist) {
+    const indexableData = (await Promise.all(
+      indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {
+        const results = await archivist.get([imageThumbnailHash, timestampHash]);
+        const imageThumbnailPayload = results.find(import_image_thumbnail_payload_plugin.isImageThumbnail);
+        const timestampPayload = results.find(import_witness_timestamp.isTimestamp);
+        if (!imageThumbnailPayload || !timestampPayload) {
+          console.log("Could not find payload from hash to index");
+          return void 0;
+        }
+        const calculatedImageThumbnailHash = await import_core.PayloadHasher.hashAsync(imageThumbnailPayload);
+        const calculatedTimestampHash = await import_core.PayloadHasher.hashAsync(timestampPayload);
+        if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash)
+          return void 0;
+        return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload];
+      })
+    )).filter(import_exists.exists);
+    return indexableData;
+  }
+  static async indexableHashes(batch) {
+    const indexableHashes = (await Promise.all(
+      batch.filter(import_boundwitness_model.isBoundWitness).map(async (bw) => {
+        const imageThumbnailIndexes = bw.payload_schemas?.map((schema, index) => schema === import_image_thumbnail_payload_plugin.ImageThumbnailSchema ? index : void 0).filter(import_exists.exists);
+        const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === import_witness_timestamp.TimestampSchema);
+        if (!imageThumbnailIndexes.length || timestampIndex === -1)
+          return void 0;
+        const imageThumbnails = bw.payload_hashes.map((hash, index) => imageThumbnailIndexes.includes(index) ? hash : void 0).filter(import_exists.exists);
+        const timestampHash = bw.payload_hashes?.[timestampIndex];
+        const boundWitnessHash = await import_core.PayloadHasher.hashAsync(bw);
+        return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash]);
+      })
+    )).flat().filter(import_exists.exists);
+    return indexableHashes;
+  }
   /**
    * Works in the background to populate index for the Diviner
    * @returns
@@ -67,35 +101,9 @@ var ImageThumbnailDiviner = class extends import_abstract_diviner.AbstractDivine
     const batch = await boundWitnessDiviner.divine([query]);
     if (batch.length === 0)
       return;
-    const indexableHashes = (await Promise.all(
-      batch.filter(import_boundwitness_model.isBoundWitness).map(async (bw) => {
-        const imageThumbnailIndexes = bw.payload_schemas?.map((schema, index) => schema === import_image_thumbnail_payload_plugin.ImageThumbnailSchema ? index : void 0).filter(import_exists.exists);
-        const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === import_witness_timestamp.TimestampSchema);
-        if (!imageThumbnailIndexes.length || timestampIndex === -1)
-          return void 0;
-        const imageThumbnails = bw.payload_hashes.map((hash, index) => imageThumbnailIndexes.includes(index) ? hash : void 0).filter(import_exists.exists);
-        const timestampHash = bw.payload_hashes?.[timestampIndex];
-        const boundWitnessHash = await import_core.PayloadHasher.hashAsync(bw);
-        return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash]);
-      })
-    )).flat().filter(import_exists.exists);
+    const indexableHashes = await _ImageThumbnailDiviner.indexableHashes(batch);
     const archivist = await this.getArchivistForStore("thumbnailStore");
-    const indexableData = (await Promise.all(
-      indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {
-        const results = await archivist.get([imageThumbnailHash, timestampHash]);
-        const imageThumbnailPayload = results.find(import_image_thumbnail_payload_plugin.isImageThumbnail);
-        const timestampPayload = results.find(import_witness_timestamp.isTimestamp);
-        if (!imageThumbnailPayload || !timestampPayload) {
-          console.log("Could not find payload from hash to index");
-          return void 0;
-        }
-        const calculatedImageThumbnailHash = await import_core.PayloadHasher.hashAsync(imageThumbnailPayload);
-        const calculatedTimestampHash = await import_core.PayloadHasher.hashAsync(timestampPayload);
-        if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash)
-          return void 0;
-        return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload];
-      })
-    )).filter(import_exists.exists);
+    const indexableData = await _ImageThumbnailDiviner.indexableData(indexableHashes, archivist);
     const indexes = indexableData.map(
       ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {
         const { sourceUrl: url } = thumbnailPayload;

package/dist/browser/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts","../../src/Diviner/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { PayloadHasher } from '@xyo-network/core'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport { DivinerConfigSchema } from '@xyo-network/diviner-model'\nimport { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport {\n  ImageThumbnail,\n  ImageThumbnailDivinerConfig,\n  ImageThumbnailDivinerConfigSchema,\n  ImageThumbnailDivinerParams,\n  ImageThumbnailResult,\n  ImageThumbnailResultIndexSchema,\n  ImageThumbnailResultInfo,\n  ImageThumbnailSchema,\n  isImageThumbnail,\n  isImageThumbnailDivinerQuery,\n  isImageThumbnailResult,\n} from '@xyo-network/image-thumbnail-payload-plugin'\nimport { isModuleState, ModuleState, ModuleStateSchema, StateDictionary } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\nimport { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'\n\nexport type ImageThumbnailDivinerState = StateDictionary & {\n  offset: number\n}\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore' | 'thumbnailStore'\n\ntype ConfigStore = Extract<keyof ImageThumbnailDivinerConfig, ConfigStoreKey>\n\n/**\n * The fields that will need to be indexed on in the underlying store\n */\ntype QueryableImageThumbnailResultProperties = Extract<keyof ImageThumbnailResultInfo, 'status' | 'success' | 'timestamp' | 'url'>\n\n/**\n * The query that will be used to retrieve the results from the underlying store\n */\ntype ImageThumbnailResultQuery = PayloadDivinerQueryPayload & { schemas: [ImageThumbnailSchema] } & Pick<\n    ImageThumbnailResult,\n    QueryableImageThumbnailResultProperties\n  >\n\nconst moduleName = 'ImageThumbnailDiviner'\n\nexport class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {\n  static override configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema]\n\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_0000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works in the background to populate index for the Diviner\n   * @returns\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = (await this.retrieveState()) ?? { offset: 0 }\n    const { offset } = lastState\n    // Get next batch of results\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit,\n        offset,\n        order: 'asc',\n        payload_schemas: [ImageThumbnailSchema, TimestampSchema],\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return\n    // Find all the indexable hashes in this batch\n    type IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>\n    const indexableHashes: IndexableHashes[] = (\n      await Promise.all(\n        batch.filter(isBoundWitness).map(async (bw) => {\n          const imageThumbnailIndexes = bw.payload_schemas\n            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))\n            .filter(exists)\n          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)\n          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined\n          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)\n          const timestampHash = bw.payload_hashes?.[timestampIndex]\n          const boundWitnessHash = await PayloadHasher.hashAsync(bw)\n          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)\n        }),\n      )\n    )\n      .flat()\n      .filter(exists)\n    const archivist = await this.getArchivistForStore('thumbnailStore')\n    // Find all the indexable data associated with the indexable hashes\n    type IndexableData = Readonly<\n      [\n        boundWitnessHash: string,\n        imageThumbnailHash: string,\n        imageThumbnailPayload: ImageThumbnail,\n        timestampHash: string,\n        timestampPayload: TimeStamp,\n      ]\n    >\n    const indexableData: IndexableData[] = (\n      await Promise.all(\n        indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {\n          const results = await archivist.get([imageThumbnailHash, timestampHash])\n          const imageThumbnailPayload = results.find(isImageThumbnail)\n          const timestampPayload = results.find(isTimestamp)\n          if (!imageThumbnailPayload || !timestampPayload) {\n            console.log('Could not find payload from hash to index')\n            return undefined\n          }\n          const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload)\n          const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload)\n          if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash) return undefined\n          return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const\n        }),\n      )\n    ).filter(exists)\n    // Build index results from the indexable data\n    const indexes: ImageThumbnailResult[] = indexableData.map(\n      ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {\n        const { sourceUrl: url } = thumbnailPayload\n        const { timestamp } = timestampPayload\n        const status = thumbnailPayload.http?.status\n        //call anything with a thumbnail url a success\n        const success = !!thumbnailPayload.url\n        const sources = [boundWitnessHash, thumbnailHash, timestampHash]\n        const fields = { sources, status, success, timestamp, url }\n        const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema }).fields(fields).build()\n        return result\n      },\n    )\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    const nextOffset = offset + batch.length\n    const currentState = { ...lastState, offset: nextOffset }\n    await this.commitState(currentState)\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   */\n  protected async commitState(state: ImageThumbnailDivinerState) {\n    const archivist = await this.getArchivistForStore('stateStore')\n    const payload = new PayloadBuilder<ModuleState<ImageThumbnailDivinerState>>({ schema: ModuleStateSchema }).fields({ state }).build()\n    const [bw] = await new BoundWitnessBuilder().payloads([payload]).witness(this.account).build()\n    await archivist.insert([bw, payload])\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResult[]> {\n    const urls = payloads.filter(isImageThumbnailDivinerQuery)\n    const diviner = await this.getPayloadDivinerForStore('indexStore')\n    const results = (\n      await Promise.all(\n        urls.map(async (payload) => {\n          const { limit: payloadLimit, offset: payloadOffset, order: payloadOrder, status: payloadStatus, success: payloadSuccess, url } = payload\n          const limit = payloadLimit ?? 1\n          const order = payloadOrder ?? 'desc'\n          const offset = payloadOffset ?? 0\n          const success = payloadSuccess\n          const status = payloadStatus\n          const fields: Partial<ImageThumbnailResultQuery> = { ...{ limit, offset, order, status, success, url } }\n          const query = new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema }).fields(fields).build()\n          return await diviner.divine([query])\n        }),\n      )\n    )\n      .flat()\n      .filter(isImageThumbnailResult)\n    return results\n  }\n\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ImageThumbnailDivinerState | undefined> {\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState)\n      if (payload) {\n        return payload.state as ImageThumbnailDivinerState\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,oBAAuB;AACvB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,kBAA8B;AAC9B,wCAAgF;AAChF,2BAAoC;AACpC,mCAAsE;AACtE,6BAA+B;AAC/B,4CAYO;AACP,0BAA+E;AAC/E,6BAA+B;AAE/B,+BAAwD;AAuBxD,IAAM,aAAa;AAEZ,IAAM,wBAAN,cAA+G,wCAAyB;AAAA,EAC7I,OAAgB,gBAAgB,CAAC,yEAAmC,wCAAmB;AAAA,EAE/E;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAa,MAAM,KAAK,cAAc,KAAM,EAAE,QAAQ,EAAE;AAC9D,UAAM,EAAE,OAAO,IAAI;AAEnB,UAAM,sBAAsB,MAAM,KAAK,+BAA+B,gBAAgB;AACtF,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,OAAO,KAAK;AAAA,MACZ;AAAA,MACA,OAAO;AAAA,MACP,iBAAiB,CAAC,4DAAsB,wCAAe;AAAA,IACzD,CAAC,EACA,MAAM;AACT,UAAM,QAAQ,MAAM,oBAAoB,OAAO,CAAC,KAAK,CAAC;AACtD,QAAI,MAAM,WAAW;AAAG;AAGxB,UAAM,mBACJ,MAAM,QAAQ;AAAA,MACZ,MAAM,OAAO,wCAAc,EAAE,IAAI,OAAO,OAAO;AAC7C,cAAM,wBAAwB,GAAG,iBAC7B,IAAI,CAAC,QAAQ,UAAW,WAAW,6DAAuB,QAAQ,MAAU,EAC7E,OAAO,oBAAM;AAChB,cAAM,iBAAiB,GAAG,iBAAiB,UAAU,CAAC,WAAW,WAAW,wCAAe;AAC3F,YAAI,CAAC,sBAAsB,UAAU,mBAAmB;AAAI,iBAAO;AACnE,cAAM,kBAAkB,GAAG,eAAe,IAAI,CAAC,MAAM,UAAW,sBAAsB,SAAS,KAAK,IAAI,OAAO,MAAU,EAAE,OAAO,oBAAM;AACxI,cAAM,gBAAgB,GAAG,iBAAiB,cAAc;AACxD,cAAM,mBAAmB,MAAM,0BAAc,UAAU,EAAE;AACzD,eAAO,gBAAgB,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,oBAAoB,aAAa,CAAU;AAAA,MACnH,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,oBAAM;AAChB,UAAM,YAAY,MAAM,KAAK,qBAAqB,gBAAgB;AAWlE,UAAM,iBACJ,MAAM,QAAQ;AAAA,MACZ,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,oBAAoB,aAAa,MAAM;AACnF,cAAM,UAAU,MAAM,UAAU,IAAI,CAAC,oBAAoB,aAAa,CAAC;AACvE,cAAM,wBAAwB,QAAQ,KAAK,sDAAgB;AAC3D,cAAM,mBAAmB,QAAQ,KAAK,oCAAW;AACjD,YAAI,CAAC,yBAAyB,CAAC,kBAAkB;AAC/C,kBAAQ,IAAI,2CAA2C;AACvD,iBAAO;AAAA,QACT;AACA,cAAM,+BAA+B,MAAM,0BAAc,UAAU,qBAAqB;AACxF,cAAM,0BAA0B,MAAM,0BAAc,UAAU,gBAAgB;AAC9E,YAAI,uBAAuB,gCAAgC,kBAAkB;AAAyB,iBAAO;AAC7G,eAAO,CAAC,kBAAkB,oBAAoB,uBAAuB,eAAe,gBAAgB;AAAA,MACtG,CAAC;AAAA,IACH,GACA,OAAO,oBAAM;AAEf,UAAM,UAAkC,cAAc;AAAA,MACpD,CAAC,CAAC,kBAAkB,eAAe,kBAAkB,eAAe,gBAAgB,MAAM;AACxF,cAAM,EAAE,WAAW,IAAI,IAAI;AAC3B,cAAM,EAAE,UAAU,IAAI;AACtB,cAAM,SAAS,iBAAiB,MAAM;AAEtC,cAAM,UAAU,CAAC,CAAC,iBAAiB;AACnC,cAAM,UAAU,CAAC,kBAAkB,eAAe,aAAa;AAC/D,cAAM,SAAS,EAAE,SAAS,QAAQ,SAAS,WAAW,IAAI;AAC1D,cAAM,SAAS,IAAI,sCAAqC,EAAE,QAAQ,sEAAgC,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AAC1H,eAAO;AAAA,MACT;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,UAAM,aAAa,SAAS,MAAM;AAClC,UAAM,eAAe,EAAE,GAAG,WAAW,QAAQ,WAAW;AACxD,UAAM,KAAK,YAAY,YAAY;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAgB,YAAY,OAAmC;AAC7D,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,UAAU,IAAI,sCAAwD,EAAE,QAAQ,sCAAkB,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM;AACnI,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC7F,UAAM,UAAU,OAAO,CAAC,IAAI,OAAO,CAAC;AAAA,EACtC;AAAA,EAEA,MAAyB,cAAc,WAAsB,CAAC,GAAoC;AAChG,UAAM,OAAO,SAAS,OAAO,kEAA4B;AACzD,UAAM,UAAU,MAAM,KAAK,0BAA0B,YAAY;AACjE,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,KAAK,IAAI,OAAO,YAAY;AAC1B,cAAM,EAAE,OAAO,cAAc,QAAQ,eAAe,OAAO,cAAc,QAAQ,eAAe,SAAS,gBAAgB,IAAI,IAAI;AACjI,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,SAAS,iBAAiB;AAChC,cAAM,UAAU;AAChB,cAAM,SAAS;AACf,cAAM,SAA6C,EAAE,GAAG,EAAE,OAAO,QAAQ,OAAO,QAAQ,SAAS,IAAI,EAAE;AACvG,cAAM,QAAQ,IAAI,sCAA0C,EAAE,QAAQ,uDAA0B,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AACxH,eAAO,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AAAA,MACrC,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,4DAAsB;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA,EAEA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA,EAEA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAiE;AAC/E,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,wCAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAa;AAChE,UAAI,SAAS;AACX,eAAO,QAAQ;AAAA,MACjB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA,EAEQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
+{"version":3,"sources":["../../src/index.ts","../../src/Diviner/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistInstance } from '@xyo-network/archivist-model'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { PayloadHasher } from '@xyo-network/core'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport { DivinerConfigSchema } from '@xyo-network/diviner-model'\nimport { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport {\n  ImageThumbnail,\n  ImageThumbnailDivinerConfig,\n  ImageThumbnailDivinerConfigSchema,\n  ImageThumbnailDivinerParams,\n  ImageThumbnailResult,\n  ImageThumbnailResultIndexSchema,\n  ImageThumbnailResultInfo,\n  ImageThumbnailSchema,\n  isImageThumbnail,\n  isImageThumbnailDivinerQuery,\n  isImageThumbnailResult,\n} from '@xyo-network/image-thumbnail-payload-plugin'\nimport { isModuleState, ModuleState, ModuleStateSchema, StateDictionary } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\nimport { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'\n\nexport type ImageThumbnailDivinerState = StateDictionary & {\n  offset: number\n}\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore' | 'thumbnailStore'\n\ntype ConfigStore = Extract<keyof ImageThumbnailDivinerConfig, ConfigStoreKey>\n\n/**\n * The fields that will need to be indexed on in the underlying store\n */\ntype QueryableImageThumbnailResultProperties = Extract<keyof ImageThumbnailResultInfo, 'status' | 'success' | 'timestamp' | 'url'>\n\n/**\n * The query that will be used to retrieve the results from the underlying store\n */\ntype ImageThumbnailResultQuery = PayloadDivinerQueryPayload & { schemas: [ImageThumbnailSchema] } & Pick<\n    ImageThumbnailResult,\n    QueryableImageThumbnailResultProperties\n  >\n\ntype IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>\n\nconst moduleName = 'ImageThumbnailDiviner'\n\nexport class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {\n  static override configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema]\n\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_0000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  private static async indexableData(indexableHashes: IndexableHashes[], archivist: ArchivistInstance) {\n    // Find all the indexable data associated with the indexable hashes\n    type IndexableData = Readonly<\n      [\n        boundWitnessHash: string,\n        imageThumbnailHash: string,\n        imageThumbnailPayload: ImageThumbnail,\n        timestampHash: string,\n        timestampPayload: TimeStamp,\n      ]\n    >\n    const indexableData: IndexableData[] = (\n      await Promise.all(\n        indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {\n          const results = await archivist.get([imageThumbnailHash, timestampHash])\n          const imageThumbnailPayload = results.find(isImageThumbnail)\n          const timestampPayload = results.find(isTimestamp)\n          if (!imageThumbnailPayload || !timestampPayload) {\n            console.log('Could not find payload from hash to index')\n            return undefined\n          }\n          const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload)\n          const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload)\n          if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash) return undefined\n          return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const\n        }),\n      )\n    ).filter(exists)\n    return indexableData\n  }\n\n  private static async indexableHashes(batch: Payload[]) {\n    // Find all the indexable hashes in this batch\n    const indexableHashes: IndexableHashes[] = (\n      await Promise.all(\n        batch.filter(isBoundWitness).map(async (bw) => {\n          const imageThumbnailIndexes = bw.payload_schemas\n            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))\n            .filter(exists)\n          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)\n          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined\n          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)\n          const timestampHash = bw.payload_hashes?.[timestampIndex]\n          const boundWitnessHash = await PayloadHasher.hashAsync(bw)\n          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)\n        }),\n      )\n    )\n      .flat()\n      .filter(exists)\n    return indexableHashes\n  }\n\n  /**\n   * Works in the background to populate index for the Diviner\n   * @returns\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = (await this.retrieveState()) ?? { offset: 0 }\n    const { offset } = lastState\n    // Get next batch of results\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit,\n        offset,\n        order: 'asc',\n        payload_schemas: [ImageThumbnailSchema, TimestampSchema],\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return\n    const indexableHashes = await ImageThumbnailDiviner.indexableHashes(batch)\n    const archivist = await this.getArchivistForStore('thumbnailStore')\n    const indexableData = await ImageThumbnailDiviner.indexableData(indexableHashes, archivist)\n    // Build index results from the indexable data\n    const indexes: ImageThumbnailResult[] = indexableData.map(\n      ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {\n        const { sourceUrl: url } = thumbnailPayload\n        const { timestamp } = timestampPayload\n        const status = thumbnailPayload.http?.status\n        //call anything with a thumbnail url a success\n        const success = !!thumbnailPayload.url\n        const sources = [boundWitnessHash, thumbnailHash, timestampHash]\n        const fields = { sources, status, success, timestamp, url }\n        const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema }).fields(fields).build()\n        return result\n      },\n    )\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    const nextOffset = offset + batch.length\n    const currentState = { ...lastState, offset: nextOffset }\n    await this.commitState(currentState)\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   */\n  protected async commitState(state: ImageThumbnailDivinerState) {\n    const archivist = await this.getArchivistForStore('stateStore')\n    const payload = new PayloadBuilder<ModuleState<ImageThumbnailDivinerState>>({ schema: ModuleStateSchema }).fields({ state }).build()\n    const [bw] = await new BoundWitnessBuilder().payloads([payload]).witness(this.account).build()\n    await archivist.insert([bw, payload])\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResult[]> {\n    const urls = payloads.filter(isImageThumbnailDivinerQuery)\n    const diviner = await this.getPayloadDivinerForStore('indexStore')\n    const results = (\n      await Promise.all(\n        urls.map(async (payload) => {\n          const { limit: payloadLimit, offset: payloadOffset, order: payloadOrder, status: payloadStatus, success: payloadSuccess, url } = payload\n          const limit = payloadLimit ?? 1\n          const order = payloadOrder ?? 'desc'\n          const offset = payloadOffset ?? 0\n          const success = payloadSuccess\n          const status = payloadStatus\n          const fields: Partial<ImageThumbnailResultQuery> = { ...{ limit, offset, order, status, success, url } }\n          const query = new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema }).fields(fields).build()\n          return await diviner.divine([query])\n        }),\n      )\n    )\n      .flat()\n      .filter(isImageThumbnailResult)\n    return results\n  }\n\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ImageThumbnailDivinerState | undefined> {\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState)\n      if (payload) {\n        return payload.state as ImageThumbnailDivinerState\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,oBAAuB;AACvB,8BAAgC;AAEhC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,kBAA8B;AAC9B,wCAAgF;AAChF,2BAAoC;AACpC,mCAAsE;AACtE,6BAA+B;AAC/B,4CAYO;AACP,0BAA+E;AAC/E,6BAA+B;AAE/B,+BAAwD;AAyBxD,IAAM,aAAa;AAEZ,IAAM,wBAAN,MAAM,+BAAyG,wCAAyB;AAAA,EAC7I,OAAgB,gBAAgB,CAAC,yEAAmC,wCAAmB;AAAA,EAE/E;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA,EAEA,aAAqB,cAAc,iBAAoC,WAA8B;AAWnG,UAAM,iBACJ,MAAM,QAAQ;AAAA,MACZ,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,oBAAoB,aAAa,MAAM;AACnF,cAAM,UAAU,MAAM,UAAU,IAAI,CAAC,oBAAoB,aAAa,CAAC;AACvE,cAAM,wBAAwB,QAAQ,KAAK,sDAAgB;AAC3D,cAAM,mBAAmB,QAAQ,KAAK,oCAAW;AACjD,YAAI,CAAC,yBAAyB,CAAC,kBAAkB;AAC/C,kBAAQ,IAAI,2CAA2C;AACvD,iBAAO;AAAA,QACT;AACA,cAAM,+BAA+B,MAAM,0BAAc,UAAU,qBAAqB;AACxF,cAAM,0BAA0B,MAAM,0BAAc,UAAU,gBAAgB;AAC9E,YAAI,uBAAuB,gCAAgC,kBAAkB;AAAyB,iBAAO;AAC7G,eAAO,CAAC,kBAAkB,oBAAoB,uBAAuB,eAAe,gBAAgB;AAAA,MACtG,CAAC;AAAA,IACH,GACA,OAAO,oBAAM;AACf,WAAO;AAAA,EACT;AAAA,EAEA,aAAqB,gBAAgB,OAAkB;AAErD,UAAM,mBACJ,MAAM,QAAQ;AAAA,MACZ,MAAM,OAAO,wCAAc,EAAE,IAAI,OAAO,OAAO;AAC7C,cAAM,wBAAwB,GAAG,iBAC7B,IAAI,CAAC,QAAQ,UAAW,WAAW,6DAAuB,QAAQ,MAAU,EAC7E,OAAO,oBAAM;AAChB,cAAM,iBAAiB,GAAG,iBAAiB,UAAU,CAAC,WAAW,WAAW,wCAAe;AAC3F,YAAI,CAAC,sBAAsB,UAAU,mBAAmB;AAAI,iBAAO;AACnE,cAAM,kBAAkB,GAAG,eAAe,IAAI,CAAC,MAAM,UAAW,sBAAsB,SAAS,KAAK,IAAI,OAAO,MAAU,EAAE,OAAO,oBAAM;AACxI,cAAM,gBAAgB,GAAG,iBAAiB,cAAc;AACxD,cAAM,mBAAmB,MAAM,0BAAc,UAAU,EAAE;AACzD,eAAO,gBAAgB,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,oBAAoB,aAAa,CAAU;AAAA,MACnH,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,oBAAM;AAChB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAa,MAAM,KAAK,cAAc,KAAM,EAAE,QAAQ,EAAE;AAC9D,UAAM,EAAE,OAAO,IAAI;AAEnB,UAAM,sBAAsB,MAAM,KAAK,+BAA+B,gBAAgB;AACtF,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,OAAO,KAAK;AAAA,MACZ;AAAA,MACA,OAAO;AAAA,MACP,iBAAiB,CAAC,4DAAsB,wCAAe;AAAA,IACzD,CAAC,EACA,MAAM;AACT,UAAM,QAAQ,MAAM,oBAAoB,OAAO,CAAC,KAAK,CAAC;AACtD,QAAI,MAAM,WAAW;AAAG;AACxB,UAAM,kBAAkB,MAAM,uBAAsB,gBAAgB,KAAK;AACzE,UAAM,YAAY,MAAM,KAAK,qBAAqB,gBAAgB;AAClE,UAAM,gBAAgB,MAAM,uBAAsB,cAAc,iBAAiB,SAAS;AAE1F,UAAM,UAAkC,cAAc;AAAA,MACpD,CAAC,CAAC,kBAAkB,eAAe,kBAAkB,eAAe,gBAAgB,MAAM;AACxF,cAAM,EAAE,WAAW,IAAI,IAAI;AAC3B,cAAM,EAAE,UAAU,IAAI;AACtB,cAAM,SAAS,iBAAiB,MAAM;AAEtC,cAAM,UAAU,CAAC,CAAC,iBAAiB;AACnC,cAAM,UAAU,CAAC,kBAAkB,eAAe,aAAa;AAC/D,cAAM,SAAS,EAAE,SAAS,QAAQ,SAAS,WAAW,IAAI;AAC1D,cAAM,SAAS,IAAI,sCAAqC,EAAE,QAAQ,sEAAgC,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AAC1H,eAAO;AAAA,MACT;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,UAAM,aAAa,SAAS,MAAM;AAClC,UAAM,eAAe,EAAE,GAAG,WAAW,QAAQ,WAAW;AACxD,UAAM,KAAK,YAAY,YAAY;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAgB,YAAY,OAAmC;AAC7D,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,UAAU,IAAI,sCAAwD,EAAE,QAAQ,sCAAkB,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM;AACnI,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC7F,UAAM,UAAU,OAAO,CAAC,IAAI,OAAO,CAAC;AAAA,EACtC;AAAA,EAEA,MAAyB,cAAc,WAAsB,CAAC,GAAoC;AAChG,UAAM,OAAO,SAAS,OAAO,kEAA4B;AACzD,UAAM,UAAU,MAAM,KAAK,0BAA0B,YAAY;AACjE,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,KAAK,IAAI,OAAO,YAAY;AAC1B,cAAM,EAAE,OAAO,cAAc,QAAQ,eAAe,OAAO,cAAc,QAAQ,eAAe,SAAS,gBAAgB,IAAI,IAAI;AACjI,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,SAAS,iBAAiB;AAChC,cAAM,UAAU;AAChB,cAAM,SAAS;AACf,cAAM,SAA6C,EAAE,GAAG,EAAE,OAAO,QAAQ,OAAO,QAAQ,SAAS,IAAI,EAAE;AACvG,cAAM,QAAQ,IAAI,sCAA0C,EAAE,QAAQ,uDAA0B,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AACxH,eAAO,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AAAA,MACrC,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,4DAAsB;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA,EAEA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA,EAEA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAiE;AAC/E,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,wCAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAa;AAChE,UAAI,SAAS;AACX,eAAO,QAAQ;AAAA,MACjB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA,EAEQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}

package/dist/browser/index.js

@@ -22,7 +22,7 @@ import { isModuleState, ModuleStateSchema } from "@xyo-network/module-model";
 import { PayloadBuilder } from "@xyo-network/payload-builder";
 import { isTimestamp, TimestampSchema } from "@xyo-network/witness-timestamp";
 var moduleName = "ImageThumbnailDiviner";
-var ImageThumbnailDiviner = class extends AbstractDiviner {
+var ImageThumbnailDiviner = class _ImageThumbnailDiviner extends AbstractDiviner {
   static configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema];
   _pollId;
   get payloadDivinerLimit() {
@@ -31,6 +31,40 @@ var ImageThumbnailDiviner = class extends AbstractDiviner {
   get pollFrequency() {
     return this.config.pollFrequency ?? 1e4;
   }
+  static async indexableData(indexableHashes, archivist) {
+    const indexableData = (await Promise.all(
+      indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {
+        const results = await archivist.get([imageThumbnailHash, timestampHash]);
+        const imageThumbnailPayload = results.find(isImageThumbnail);
+        const timestampPayload = results.find(isTimestamp);
+        if (!imageThumbnailPayload || !timestampPayload) {
+          console.log("Could not find payload from hash to index");
+          return void 0;
+        }
+        const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload);
+        const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload);
+        if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash)
+          return void 0;
+        return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload];
+      })
+    )).filter(exists);
+    return indexableData;
+  }
+  static async indexableHashes(batch) {
+    const indexableHashes = (await Promise.all(
+      batch.filter(isBoundWitness).map(async (bw) => {
+        const imageThumbnailIndexes = bw.payload_schemas?.map((schema, index) => schema === ImageThumbnailSchema ? index : void 0).filter(exists);
+        const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema);
+        if (!imageThumbnailIndexes.length || timestampIndex === -1)
+          return void 0;
+        const imageThumbnails = bw.payload_hashes.map((hash, index) => imageThumbnailIndexes.includes(index) ? hash : void 0).filter(exists);
+        const timestampHash = bw.payload_hashes?.[timestampIndex];
+        const boundWitnessHash = await PayloadHasher.hashAsync(bw);
+        return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash]);
+      })
+    )).flat().filter(exists);
+    return indexableHashes;
+  }
   /**
    * Works in the background to populate index for the Diviner
    * @returns
@@ -48,35 +82,9 @@ var ImageThumbnailDiviner = class extends AbstractDiviner {
     const batch = await boundWitnessDiviner.divine([query]);
     if (batch.length === 0)
       return;
-    const indexableHashes = (await Promise.all(
-      batch.filter(isBoundWitness).map(async (bw) => {
-        const imageThumbnailIndexes = bw.payload_schemas?.map((schema, index) => schema === ImageThumbnailSchema ? index : void 0).filter(exists);
-        const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema);
-        if (!imageThumbnailIndexes.length || timestampIndex === -1)
-          return void 0;
-        const imageThumbnails = bw.payload_hashes.map((hash, index) => imageThumbnailIndexes.includes(index) ? hash : void 0).filter(exists);
-        const timestampHash = bw.payload_hashes?.[timestampIndex];
-        const boundWitnessHash = await PayloadHasher.hashAsync(bw);
-        return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash]);
-      })
-    )).flat().filter(exists);
+    const indexableHashes = await _ImageThumbnailDiviner.indexableHashes(batch);
     const archivist = await this.getArchivistForStore("thumbnailStore");
-    const indexableData = (await Promise.all(
-      indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {
-        const results = await archivist.get([imageThumbnailHash, timestampHash]);
-        const imageThumbnailPayload = results.find(isImageThumbnail);
-        const timestampPayload = results.find(isTimestamp);
-        if (!imageThumbnailPayload || !timestampPayload) {
-          console.log("Could not find payload from hash to index");
-          return void 0;
-        }
-        const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload);
-        const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload);
-        if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash)
-          return void 0;
-        return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload];
-      })
-    )).filter(exists);
+    const indexableData = await _ImageThumbnailDiviner.indexableData(indexableHashes, archivist);
     const indexes = indexableData.map(
       ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {
         const { sourceUrl: url } = thumbnailPayload;

package/dist/browser/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { PayloadHasher } from '@xyo-network/core'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport { DivinerConfigSchema } from '@xyo-network/diviner-model'\nimport { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport {\n  ImageThumbnail,\n  ImageThumbnailDivinerConfig,\n  ImageThumbnailDivinerConfigSchema,\n  ImageThumbnailDivinerParams,\n  ImageThumbnailResult,\n  ImageThumbnailResultIndexSchema,\n  ImageThumbnailResultInfo,\n  ImageThumbnailSchema,\n  isImageThumbnail,\n  isImageThumbnailDivinerQuery,\n  isImageThumbnailResult,\n} from '@xyo-network/image-thumbnail-payload-plugin'\nimport { isModuleState, ModuleState, ModuleStateSchema, StateDictionary } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\nimport { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'\n\nexport type ImageThumbnailDivinerState = StateDictionary & {\n  offset: number\n}\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore' | 'thumbnailStore'\n\ntype ConfigStore = Extract<keyof ImageThumbnailDivinerConfig, ConfigStoreKey>\n\n/**\n * The fields that will need to be indexed on in the underlying store\n */\ntype QueryableImageThumbnailResultProperties = Extract<keyof ImageThumbnailResultInfo, 'status' | 'success' | 'timestamp' | 'url'>\n\n/**\n * The query that will be used to retrieve the results from the underlying store\n */\ntype ImageThumbnailResultQuery = PayloadDivinerQueryPayload & { schemas: [ImageThumbnailSchema] } & Pick<\n    ImageThumbnailResult,\n    QueryableImageThumbnailResultProperties\n  >\n\nconst moduleName = 'ImageThumbnailDiviner'\n\nexport class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {\n  static override configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema]\n\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_0000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works in the background to populate index for the Diviner\n   * @returns\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = (await this.retrieveState()) ?? { offset: 0 }\n    const { offset } = lastState\n    // Get next batch of results\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit,\n        offset,\n        order: 'asc',\n        payload_schemas: [ImageThumbnailSchema, TimestampSchema],\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return\n    // Find all the indexable hashes in this batch\n    type IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>\n    const indexableHashes: IndexableHashes[] = (\n      await Promise.all(\n        batch.filter(isBoundWitness).map(async (bw) => {\n          const imageThumbnailIndexes = bw.payload_schemas\n            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))\n            .filter(exists)\n          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)\n          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined\n          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)\n          const timestampHash = bw.payload_hashes?.[timestampIndex]\n          const boundWitnessHash = await PayloadHasher.hashAsync(bw)\n          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)\n        }),\n      )\n    )\n      .flat()\n      .filter(exists)\n    const archivist = await this.getArchivistForStore('thumbnailStore')\n    // Find all the indexable data associated with the indexable hashes\n    type IndexableData = Readonly<\n      [\n        boundWitnessHash: string,\n        imageThumbnailHash: string,\n        imageThumbnailPayload: ImageThumbnail,\n        timestampHash: string,\n        timestampPayload: TimeStamp,\n      ]\n    >\n    const indexableData: IndexableData[] = (\n      await Promise.all(\n        indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {\n          const results = await archivist.get([imageThumbnailHash, timestampHash])\n          const imageThumbnailPayload = results.find(isImageThumbnail)\n          const timestampPayload = results.find(isTimestamp)\n          if (!imageThumbnailPayload || !timestampPayload) {\n            console.log('Could not find payload from hash to index')\n            return undefined\n          }\n          const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload)\n          const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload)\n          if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash) return undefined\n          return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const\n        }),\n      )\n    ).filter(exists)\n    // Build index results from the indexable data\n    const indexes: ImageThumbnailResult[] = indexableData.map(\n      ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {\n        const { sourceUrl: url } = thumbnailPayload\n        const { timestamp } = timestampPayload\n        const status = thumbnailPayload.http?.status\n        //call anything with a thumbnail url a success\n        const success = !!thumbnailPayload.url\n        const sources = [boundWitnessHash, thumbnailHash, timestampHash]\n        const fields = { sources, status, success, timestamp, url }\n        const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema }).fields(fields).build()\n        return result\n      },\n    )\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    const nextOffset = offset + batch.length\n    const currentState = { ...lastState, offset: nextOffset }\n    await this.commitState(currentState)\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   */\n  protected async commitState(state: ImageThumbnailDivinerState) {\n    const archivist = await this.getArchivistForStore('stateStore')\n    const payload = new PayloadBuilder<ModuleState<ImageThumbnailDivinerState>>({ schema: ModuleStateSchema }).fields({ state }).build()\n    const [bw] = await new BoundWitnessBuilder().payloads([payload]).witness(this.account).build()\n    await archivist.insert([bw, payload])\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResult[]> {\n    const urls = payloads.filter(isImageThumbnailDivinerQuery)\n    const diviner = await this.getPayloadDivinerForStore('indexStore')\n    const results = (\n      await Promise.all(\n        urls.map(async (payload) => {\n          const { limit: payloadLimit, offset: payloadOffset, order: payloadOrder, status: payloadStatus, success: payloadSuccess, url } = payload\n          const limit = payloadLimit ?? 1\n          const order = payloadOrder ?? 'desc'\n          const offset = payloadOffset ?? 0\n          const success = payloadSuccess\n          const status = payloadStatus\n          const fields: Partial<ImageThumbnailResultQuery> = { ...{ limit, offset, order, status, success, url } }\n          const query = new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema }).fields(fields).build()\n          return await diviner.divine([query])\n        }),\n      )\n    )\n      .flat()\n      .filter(isImageThumbnailResult)\n    return results\n  }\n\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ImageThumbnailDivinerState | undefined> {\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState)\n      if (payload) {\n        return payload.state as ImageThumbnailDivinerState\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,cAAc;AACvB,SAAS,uBAAuB;AAChC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAAS,qBAAqB;AAC9B,SAA0C,sCAAsC;AAChF,SAAS,2BAA2B;AACpC,SAAqC,iCAAiC;AACtE,SAAS,sBAAsB;AAC/B;AAAA,EAGE;AAAA,EAGA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,eAA4B,yBAA0C;AAC/E,SAAS,sBAAsB;AAE/B,SAAS,aAAwB,uBAAuB;AAuBxD,IAAM,aAAa;AAEZ,IAAM,wBAAN,cAA+G,gBAAyB;AAAA,EAC7I,OAAgB,gBAAgB,CAAC,mCAAmC,mBAAmB;AAAA,EAE/E;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAa,MAAM,KAAK,cAAc,KAAM,EAAE,QAAQ,EAAE;AAC9D,UAAM,EAAE,OAAO,IAAI;AAEnB,UAAM,sBAAsB,MAAM,KAAK,+BAA+B,gBAAgB;AACtF,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,OAAO,KAAK;AAAA,MACZ;AAAA,MACA,OAAO;AAAA,MACP,iBAAiB,CAAC,sBAAsB,eAAe;AAAA,IACzD,CAAC,EACA,MAAM;AACT,UAAM,QAAQ,MAAM,oBAAoB,OAAO,CAAC,KAAK,CAAC;AACtD,QAAI,MAAM,WAAW;AAAG;AAGxB,UAAM,mBACJ,MAAM,QAAQ;AAAA,MACZ,MAAM,OAAO,cAAc,EAAE,IAAI,OAAO,OAAO;AAC7C,cAAM,wBAAwB,GAAG,iBAC7B,IAAI,CAAC,QAAQ,UAAW,WAAW,uBAAuB,QAAQ,MAAU,EAC7E,OAAO,MAAM;AAChB,cAAM,iBAAiB,GAAG,iBAAiB,UAAU,CAAC,WAAW,WAAW,eAAe;AAC3F,YAAI,CAAC,sBAAsB,UAAU,mBAAmB;AAAI,iBAAO;AACnE,cAAM,kBAAkB,GAAG,eAAe,IAAI,CAAC,MAAM,UAAW,sBAAsB,SAAS,KAAK,IAAI,OAAO,MAAU,EAAE,OAAO,MAAM;AACxI,cAAM,gBAAgB,GAAG,iBAAiB,cAAc;AACxD,cAAM,mBAAmB,MAAM,cAAc,UAAU,EAAE;AACzD,eAAO,gBAAgB,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,oBAAoB,aAAa,CAAU;AAAA,MACnH,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,MAAM;AAChB,UAAM,YAAY,MAAM,KAAK,qBAAqB,gBAAgB;AAWlE,UAAM,iBACJ,MAAM,QAAQ;AAAA,MACZ,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,oBAAoB,aAAa,MAAM;AACnF,cAAM,UAAU,MAAM,UAAU,IAAI,CAAC,oBAAoB,aAAa,CAAC;AACvE,cAAM,wBAAwB,QAAQ,KAAK,gBAAgB;AAC3D,cAAM,mBAAmB,QAAQ,KAAK,WAAW;AACjD,YAAI,CAAC,yBAAyB,CAAC,kBAAkB;AAC/C,kBAAQ,IAAI,2CAA2C;AACvD,iBAAO;AAAA,QACT;AACA,cAAM,+BAA+B,MAAM,cAAc,UAAU,qBAAqB;AACxF,cAAM,0BAA0B,MAAM,cAAc,UAAU,gBAAgB;AAC9E,YAAI,uBAAuB,gCAAgC,kBAAkB;AAAyB,iBAAO;AAC7G,eAAO,CAAC,kBAAkB,oBAAoB,uBAAuB,eAAe,gBAAgB;AAAA,MACtG,CAAC;AAAA,IACH,GACA,OAAO,MAAM;AAEf,UAAM,UAAkC,cAAc;AAAA,MACpD,CAAC,CAAC,kBAAkB,eAAe,kBAAkB,eAAe,gBAAgB,MAAM;AACxF,cAAM,EAAE,WAAW,IAAI,IAAI;AAC3B,cAAM,EAAE,UAAU,IAAI;AACtB,cAAM,SAAS,iBAAiB,MAAM;AAEtC,cAAM,UAAU,CAAC,CAAC,iBAAiB;AACnC,cAAM,UAAU,CAAC,kBAAkB,eAAe,aAAa;AAC/D,cAAM,SAAS,EAAE,SAAS,QAAQ,SAAS,WAAW,IAAI;AAC1D,cAAM,SAAS,IAAI,eAAqC,EAAE,QAAQ,gCAAgC,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AAC1H,eAAO;AAAA,MACT;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,UAAM,aAAa,SAAS,MAAM;AAClC,UAAM,eAAe,EAAE,GAAG,WAAW,QAAQ,WAAW;AACxD,UAAM,KAAK,YAAY,YAAY;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAgB,YAAY,OAAmC;AAC7D,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,UAAU,IAAI,eAAwD,EAAE,QAAQ,kBAAkB,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM;AACnI,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC7F,UAAM,UAAU,OAAO,CAAC,IAAI,OAAO,CAAC;AAAA,EACtC;AAAA,EAEA,MAAyB,cAAc,WAAsB,CAAC,GAAoC;AAChG,UAAM,OAAO,SAAS,OAAO,4BAA4B;AACzD,UAAM,UAAU,MAAM,KAAK,0BAA0B,YAAY;AACjE,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,KAAK,IAAI,OAAO,YAAY;AAC1B,cAAM,EAAE,OAAO,cAAc,QAAQ,eAAe,OAAO,cAAc,QAAQ,eAAe,SAAS,gBAAgB,IAAI,IAAI;AACjI,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,SAAS,iBAAiB;AAChC,cAAM,UAAU;AAChB,cAAM,SAAS;AACf,cAAM,SAA6C,EAAE,GAAG,EAAE,OAAO,QAAQ,OAAO,QAAQ,SAAS,IAAI,EAAE;AACvG,cAAM,QAAQ,IAAI,eAA0C,EAAE,QAAQ,0BAA0B,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AACxH,eAAO,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AAAA,MACrC,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,sBAAsB;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA,EAEA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA,EAEA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAiE;AAC/E,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,oBAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAa;AAChE,UAAI,SAAS;AACX,eAAO,QAAQ;AAAA,MACjB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA,EAEQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
+{"version":3,"sources":["../../src/Diviner/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistInstance } from '@xyo-network/archivist-model'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { PayloadHasher } from '@xyo-network/core'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport { DivinerConfigSchema } from '@xyo-network/diviner-model'\nimport { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport {\n  ImageThumbnail,\n  ImageThumbnailDivinerConfig,\n  ImageThumbnailDivinerConfigSchema,\n  ImageThumbnailDivinerParams,\n  ImageThumbnailResult,\n  ImageThumbnailResultIndexSchema,\n  ImageThumbnailResultInfo,\n  ImageThumbnailSchema,\n  isImageThumbnail,\n  isImageThumbnailDivinerQuery,\n  isImageThumbnailResult,\n} from '@xyo-network/image-thumbnail-payload-plugin'\nimport { isModuleState, ModuleState, ModuleStateSchema, StateDictionary } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\nimport { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'\n\nexport type ImageThumbnailDivinerState = StateDictionary & {\n  offset: number\n}\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore' | 'thumbnailStore'\n\ntype ConfigStore = Extract<keyof ImageThumbnailDivinerConfig, ConfigStoreKey>\n\n/**\n * The fields that will need to be indexed on in the underlying store\n */\ntype QueryableImageThumbnailResultProperties = Extract<keyof ImageThumbnailResultInfo, 'status' | 'success' | 'timestamp' | 'url'>\n\n/**\n * The query that will be used to retrieve the results from the underlying store\n */\ntype ImageThumbnailResultQuery = PayloadDivinerQueryPayload & { schemas: [ImageThumbnailSchema] } & Pick<\n    ImageThumbnailResult,\n    QueryableImageThumbnailResultProperties\n  >\n\ntype IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>\n\nconst moduleName = 'ImageThumbnailDiviner'\n\nexport class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {\n  static override configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema]\n\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_0000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  private static async indexableData(indexableHashes: IndexableHashes[], archivist: ArchivistInstance) {\n    // Find all the indexable data associated with the indexable hashes\n    type IndexableData = Readonly<\n      [\n        boundWitnessHash: string,\n        imageThumbnailHash: string,\n        imageThumbnailPayload: ImageThumbnail,\n        timestampHash: string,\n        timestampPayload: TimeStamp,\n      ]\n    >\n    const indexableData: IndexableData[] = (\n      await Promise.all(\n        indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {\n          const results = await archivist.get([imageThumbnailHash, timestampHash])\n          const imageThumbnailPayload = results.find(isImageThumbnail)\n          const timestampPayload = results.find(isTimestamp)\n          if (!imageThumbnailPayload || !timestampPayload) {\n            console.log('Could not find payload from hash to index')\n            return undefined\n          }\n          const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload)\n          const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload)\n          if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash) return undefined\n          return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const\n        }),\n      )\n    ).filter(exists)\n    return indexableData\n  }\n\n  private static async indexableHashes(batch: Payload[]) {\n    // Find all the indexable hashes in this batch\n    const indexableHashes: IndexableHashes[] = (\n      await Promise.all(\n        batch.filter(isBoundWitness).map(async (bw) => {\n          const imageThumbnailIndexes = bw.payload_schemas\n            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))\n            .filter(exists)\n          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)\n          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined\n          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)\n          const timestampHash = bw.payload_hashes?.[timestampIndex]\n          const boundWitnessHash = await PayloadHasher.hashAsync(bw)\n          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)\n        }),\n      )\n    )\n      .flat()\n      .filter(exists)\n    return indexableHashes\n  }\n\n  /**\n   * Works in the background to populate index for the Diviner\n   * @returns\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = (await this.retrieveState()) ?? { offset: 0 }\n    const { offset } = lastState\n    // Get next batch of results\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit,\n        offset,\n        order: 'asc',\n        payload_schemas: [ImageThumbnailSchema, TimestampSchema],\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return\n    const indexableHashes = await ImageThumbnailDiviner.indexableHashes(batch)\n    const archivist = await this.getArchivistForStore('thumbnailStore')\n    const indexableData = await ImageThumbnailDiviner.indexableData(indexableHashes, archivist)\n    // Build index results from the indexable data\n    const indexes: ImageThumbnailResult[] = indexableData.map(\n      ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {\n        const { sourceUrl: url } = thumbnailPayload\n        const { timestamp } = timestampPayload\n        const status = thumbnailPayload.http?.status\n        //call anything with a thumbnail url a success\n        const success = !!thumbnailPayload.url\n        const sources = [boundWitnessHash, thumbnailHash, timestampHash]\n        const fields = { sources, status, success, timestamp, url }\n        const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema }).fields(fields).build()\n        return result\n      },\n    )\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    const nextOffset = offset + batch.length\n    const currentState = { ...lastState, offset: nextOffset }\n    await this.commitState(currentState)\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   */\n  protected async commitState(state: ImageThumbnailDivinerState) {\n    const archivist = await this.getArchivistForStore('stateStore')\n    const payload = new PayloadBuilder<ModuleState<ImageThumbnailDivinerState>>({ schema: ModuleStateSchema }).fields({ state }).build()\n    const [bw] = await new BoundWitnessBuilder().payloads([payload]).witness(this.account).build()\n    await archivist.insert([bw, payload])\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResult[]> {\n    const urls = payloads.filter(isImageThumbnailDivinerQuery)\n    const diviner = await this.getPayloadDivinerForStore('indexStore')\n    const results = (\n      await Promise.all(\n        urls.map(async (payload) => {\n          const { limit: payloadLimit, offset: payloadOffset, order: payloadOrder, status: payloadStatus, success: payloadSuccess, url } = payload\n          const limit = payloadLimit ?? 1\n          const order = payloadOrder ?? 'desc'\n          const offset = payloadOffset ?? 0\n          const success = payloadSuccess\n          const status = payloadStatus\n          const fields: Partial<ImageThumbnailResultQuery> = { ...{ limit, offset, order, status, success, url } }\n          const query = new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema }).fields(fields).build()\n          return await diviner.divine([query])\n        }),\n      )\n    )\n      .flat()\n      .filter(isImageThumbnailResult)\n    return results\n  }\n\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ImageThumbnailDivinerState | undefined> {\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState)\n      if (payload) {\n        return payload.state as ImageThumbnailDivinerState\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,cAAc;AACvB,SAAS,uBAAuB;AAEhC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAAS,qBAAqB;AAC9B,SAA0C,sCAAsC;AAChF,SAAS,2BAA2B;AACpC,SAAqC,iCAAiC;AACtE,SAAS,sBAAsB;AAC/B;AAAA,EAGE;AAAA,EAGA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,eAA4B,yBAA0C;AAC/E,SAAS,sBAAsB;AAE/B,SAAS,aAAwB,uBAAuB;AAyBxD,IAAM,aAAa;AAEZ,IAAM,wBAAN,MAAM,+BAAyG,gBAAyB;AAAA,EAC7I,OAAgB,gBAAgB,CAAC,mCAAmC,mBAAmB;AAAA,EAE/E;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA,EAEA,aAAqB,cAAc,iBAAoC,WAA8B;AAWnG,UAAM,iBACJ,MAAM,QAAQ;AAAA,MACZ,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,oBAAoB,aAAa,MAAM;AACnF,cAAM,UAAU,MAAM,UAAU,IAAI,CAAC,oBAAoB,aAAa,CAAC;AACvE,cAAM,wBAAwB,QAAQ,KAAK,gBAAgB;AAC3D,cAAM,mBAAmB,QAAQ,KAAK,WAAW;AACjD,YAAI,CAAC,yBAAyB,CAAC,kBAAkB;AAC/C,kBAAQ,IAAI,2CAA2C;AACvD,iBAAO;AAAA,QACT;AACA,cAAM,+BAA+B,MAAM,cAAc,UAAU,qBAAqB;AACxF,cAAM,0BAA0B,MAAM,cAAc,UAAU,gBAAgB;AAC9E,YAAI,uBAAuB,gCAAgC,kBAAkB;AAAyB,iBAAO;AAC7G,eAAO,CAAC,kBAAkB,oBAAoB,uBAAuB,eAAe,gBAAgB;AAAA,MACtG,CAAC;AAAA,IACH,GACA,OAAO,MAAM;AACf,WAAO;AAAA,EACT;AAAA,EAEA,aAAqB,gBAAgB,OAAkB;AAErD,UAAM,mBACJ,MAAM,QAAQ;AAAA,MACZ,MAAM,OAAO,cAAc,EAAE,IAAI,OAAO,OAAO;AAC7C,cAAM,wBAAwB,GAAG,iBAC7B,IAAI,CAAC,QAAQ,UAAW,WAAW,uBAAuB,QAAQ,MAAU,EAC7E,OAAO,MAAM;AAChB,cAAM,iBAAiB,GAAG,iBAAiB,UAAU,CAAC,WAAW,WAAW,eAAe;AAC3F,YAAI,CAAC,sBAAsB,UAAU,mBAAmB;AAAI,iBAAO;AACnE,cAAM,kBAAkB,GAAG,eAAe,IAAI,CAAC,MAAM,UAAW,sBAAsB,SAAS,KAAK,IAAI,OAAO,MAAU,EAAE,OAAO,MAAM;AACxI,cAAM,gBAAgB,GAAG,iBAAiB,cAAc;AACxD,cAAM,mBAAmB,MAAM,cAAc,UAAU,EAAE;AACzD,eAAO,gBAAgB,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,oBAAoB,aAAa,CAAU;AAAA,MACnH,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,MAAM;AAChB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAa,MAAM,KAAK,cAAc,KAAM,EAAE,QAAQ,EAAE;AAC9D,UAAM,EAAE,OAAO,IAAI;AAEnB,UAAM,sBAAsB,MAAM,KAAK,+BAA+B,gBAAgB;AACtF,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,OAAO,KAAK;AAAA,MACZ;AAAA,MACA,OAAO;AAAA,MACP,iBAAiB,CAAC,sBAAsB,eAAe;AAAA,IACzD,CAAC,EACA,MAAM;AACT,UAAM,QAAQ,MAAM,oBAAoB,OAAO,CAAC,KAAK,CAAC;AACtD,QAAI,MAAM,WAAW;AAAG;AACxB,UAAM,kBAAkB,MAAM,uBAAsB,gBAAgB,KAAK;AACzE,UAAM,YAAY,MAAM,KAAK,qBAAqB,gBAAgB;AAClE,UAAM,gBAAgB,MAAM,uBAAsB,cAAc,iBAAiB,SAAS;AAE1F,UAAM,UAAkC,cAAc;AAAA,MACpD,CAAC,CAAC,kBAAkB,eAAe,kBAAkB,eAAe,gBAAgB,MAAM;AACxF,cAAM,EAAE,WAAW,IAAI,IAAI;AAC3B,cAAM,EAAE,UAAU,IAAI;AACtB,cAAM,SAAS,iBAAiB,MAAM;AAEtC,cAAM,UAAU,CAAC,CAAC,iBAAiB;AACnC,cAAM,UAAU,CAAC,kBAAkB,eAAe,aAAa;AAC/D,cAAM,SAAS,EAAE,SAAS,QAAQ,SAAS,WAAW,IAAI;AAC1D,cAAM,SAAS,IAAI,eAAqC,EAAE,QAAQ,gCAAgC,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AAC1H,eAAO;AAAA,MACT;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,UAAM,aAAa,SAAS,MAAM;AAClC,UAAM,eAAe,EAAE,GAAG,WAAW,QAAQ,WAAW;AACxD,UAAM,KAAK,YAAY,YAAY;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAgB,YAAY,OAAmC;AAC7D,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,UAAU,IAAI,eAAwD,EAAE,QAAQ,kBAAkB,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM;AACnI,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC7F,UAAM,UAAU,OAAO,CAAC,IAAI,OAAO,CAAC;AAAA,EACtC;AAAA,EAEA,MAAyB,cAAc,WAAsB,CAAC,GAAoC;AAChG,UAAM,OAAO,SAAS,OAAO,4BAA4B;AACzD,UAAM,UAAU,MAAM,KAAK,0BAA0B,YAAY;AACjE,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,KAAK,IAAI,OAAO,YAAY;AAC1B,cAAM,EAAE,OAAO,cAAc,QAAQ,eAAe,OAAO,cAAc,QAAQ,eAAe,SAAS,gBAAgB,IAAI,IAAI;AACjI,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,SAAS,iBAAiB;AAChC,cAAM,UAAU;AAChB,cAAM,SAAS;AACf,cAAM,SAA6C,EAAE,GAAG,EAAE,OAAO,QAAQ,OAAO,QAAQ,SAAS,IAAI,EAAE;AACvG,cAAM,QAAQ,IAAI,eAA0C,EAAE,QAAQ,0BAA0B,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AACxH,eAAO,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AAAA,MACrC,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,sBAAsB;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA,EAEA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA,EAEA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAiE;AAC/E,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,oBAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAa;AAChE,UAAI,SAAS;AACX,eAAO,QAAQ;AAAA,MACjB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA,EAEQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}

package/dist/node/Diviner/Diviner.d.cts

@@ -14,6 +14,8 @@ export declare class ImageThumbnailDiviner<TParams extends ImageThumbnailDiviner
     private _pollId?;
     get payloadDivinerLimit(): number;
     get pollFrequency(): number;
+    private static indexableData;
+    private static indexableHashes;
     /**
      * Works in the background to populate index for the Diviner
      * @returns

package/dist/node/Diviner/Diviner.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAiB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAoFnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAmB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;mBAEoB,aAAa;mBA+Bb,eAAe;IAsBpC;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAwCnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}

package/dist/node/Diviner/Diviner.d.mts

@@ -14,6 +14,8 @@ export declare class ImageThumbnailDiviner<TParams extends ImageThumbnailDiviner
     private _pollId?;
     get payloadDivinerLimit(): number;
     get pollFrequency(): number;
+    private static indexableData;
+    private static indexableHashes;
     /**
      * Works in the background to populate index for the Diviner
      * @returns

package/dist/node/Diviner/Diviner.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAiB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAoFnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAmB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;mBAEoB,aAAa;mBA+Bb,eAAe;IAsBpC;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAwCnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}

package/dist/node/Diviner/Diviner.d.ts

@@ -14,6 +14,8 @@ export declare class ImageThumbnailDiviner<TParams extends ImageThumbnailDiviner
     private _pollId?;
     get payloadDivinerLimit(): number;
     get pollFrequency(): number;
+    private static indexableData;
+    private static indexableHashes;
     /**
      * Works in the background to populate index for the Diviner
      * @returns

package/dist/node/Diviner/Diviner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAiB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAoFnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}
+{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../../src/Diviner/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAOjE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAEL,2BAA2B,EAE3B,2BAA2B,EAC3B,oBAAoB,EAOrB,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAiD,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAGpD,MAAM,MAAM,0BAA0B,GAAG,eAAe,GAAG;IACzD,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAA;AAEpE,KAAK,WAAW,GAAG,OAAO,CAAC,MAAM,2BAA2B,EAAE,cAAc,CAAC,CAAA;AAmB7E,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,kFAA2D;IAExF,OAAO,CAAC,OAAO,CAAC,CAAkC;IAElD,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;mBAEoB,aAAa;mBA+Bb,eAAe;IAsBpC;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAwCnD;IAED;;;;;OAKG;cACa,WAAW,CAAC,KAAK,EAAE,0BAA0B;cAOpC,aAAa,CAAC,QAAQ,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;cAuBjF,oBAAoB,CAAC,KAAK,EAAE,WAAW;cAMvC,8BAA8B,CAAC,KAAK,EAAE,WAAW;cAMjD,yBAAyB,CAAC,KAAK,EAAE,WAAW;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC;cAuCvD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF,OAAO,CAAC,IAAI;CAab"}

package/dist/node/index.js

@@ -41,7 +41,7 @@ var import_module_model = require("@xyo-network/module-model");
 var import_payload_builder = require("@xyo-network/payload-builder");
 var import_witness_timestamp = require("@xyo-network/witness-timestamp");
 var moduleName = "ImageThumbnailDiviner";
-var ImageThumbnailDiviner = class extends import_abstract_diviner.AbstractDiviner {
+var ImageThumbnailDiviner = class _ImageThumbnailDiviner extends import_abstract_diviner.AbstractDiviner {
   static configSchemas = [import_image_thumbnail_payload_plugin.ImageThumbnailDivinerConfigSchema, import_diviner_model.DivinerConfigSchema];
   _pollId;
   get payloadDivinerLimit() {
@@ -50,6 +50,41 @@ var ImageThumbnailDiviner = class extends import_abstract_diviner.AbstractDivine
   get pollFrequency() {
     return this.config.pollFrequency ?? 1e4;
   }
+  static async indexableData(indexableHashes, archivist) {
+    const indexableData = (await Promise.all(
+      indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {
+        const results = await archivist.get([imageThumbnailHash, timestampHash]);
+        const imageThumbnailPayload = results.find(import_image_thumbnail_payload_plugin.isImageThumbnail);
+        const timestampPayload = results.find(import_witness_timestamp.isTimestamp);
+        if (!imageThumbnailPayload || !timestampPayload) {
+          console.log("Could not find payload from hash to index");
+          return void 0;
+        }
+        const calculatedImageThumbnailHash = await import_core.PayloadHasher.hashAsync(imageThumbnailPayload);
+        const calculatedTimestampHash = await import_core.PayloadHasher.hashAsync(timestampPayload);
+        if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash)
+          return void 0;
+        return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload];
+      })
+    )).filter(import_exists.exists);
+    return indexableData;
+  }
+  static async indexableHashes(batch) {
+    const indexableHashes = (await Promise.all(
+      batch.filter(import_boundwitness_model.isBoundWitness).map(async (bw) => {
+        var _a, _b, _c;
+        const imageThumbnailIndexes = (_a = bw.payload_schemas) == null ? void 0 : _a.map((schema, index) => schema === import_image_thumbnail_payload_plugin.ImageThumbnailSchema ? index : void 0).filter(import_exists.exists);
+        const timestampIndex = (_b = bw.payload_schemas) == null ? void 0 : _b.findIndex((schema) => schema === import_witness_timestamp.TimestampSchema);
+        if (!imageThumbnailIndexes.length || timestampIndex === -1)
+          return void 0;
+        const imageThumbnails = bw.payload_hashes.map((hash, index) => imageThumbnailIndexes.includes(index) ? hash : void 0).filter(import_exists.exists);
+        const timestampHash = (_c = bw.payload_hashes) == null ? void 0 : _c[timestampIndex];
+        const boundWitnessHash = await import_core.PayloadHasher.hashAsync(bw);
+        return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash]);
+      })
+    )).flat().filter(import_exists.exists);
+    return indexableHashes;
+  }
   /**
    * Works in the background to populate index for the Diviner
    * @returns
@@ -67,36 +102,9 @@ var ImageThumbnailDiviner = class extends import_abstract_diviner.AbstractDivine
     const batch = await boundWitnessDiviner.divine([query]);
     if (batch.length === 0)
       return;
-    const indexableHashes = (await Promise.all(
-      batch.filter(import_boundwitness_model.isBoundWitness).map(async (bw) => {
-        var _a, _b, _c;
-        const imageThumbnailIndexes = (_a = bw.payload_schemas) == null ? void 0 : _a.map((schema, index) => schema === import_image_thumbnail_payload_plugin.ImageThumbnailSchema ? index : void 0).filter(import_exists.exists);
-        const timestampIndex = (_b = bw.payload_schemas) == null ? void 0 : _b.findIndex((schema) => schema === import_witness_timestamp.TimestampSchema);
-        if (!imageThumbnailIndexes.length || timestampIndex === -1)
-          return void 0;
-        const imageThumbnails = bw.payload_hashes.map((hash, index) => imageThumbnailIndexes.includes(index) ? hash : void 0).filter(import_exists.exists);
-        const timestampHash = (_c = bw.payload_hashes) == null ? void 0 : _c[timestampIndex];
-        const boundWitnessHash = await import_core.PayloadHasher.hashAsync(bw);
-        return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash]);
-      })
-    )).flat().filter(import_exists.exists);
+    const indexableHashes = await _ImageThumbnailDiviner.indexableHashes(batch);
     const archivist = await this.getArchivistForStore("thumbnailStore");
-    const indexableData = (await Promise.all(
-      indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {
-        const results = await archivist.get([imageThumbnailHash, timestampHash]);
-        const imageThumbnailPayload = results.find(import_image_thumbnail_payload_plugin.isImageThumbnail);
-        const timestampPayload = results.find(import_witness_timestamp.isTimestamp);
-        if (!imageThumbnailPayload || !timestampPayload) {
-          console.log("Could not find payload from hash to index");
-          return void 0;
-        }
-        const calculatedImageThumbnailHash = await import_core.PayloadHasher.hashAsync(imageThumbnailPayload);
-        const calculatedTimestampHash = await import_core.PayloadHasher.hashAsync(timestampPayload);
-        if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash)
-          return void 0;
-        return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload];
-      })
-    )).filter(import_exists.exists);
+    const indexableData = await _ImageThumbnailDiviner.indexableData(indexableHashes, archivist);
     const indexes = indexableData.map(
       ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {
         var _a;

package/dist/node/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts","../../src/Diviner/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { PayloadHasher } from '@xyo-network/core'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport { DivinerConfigSchema } from '@xyo-network/diviner-model'\nimport { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport {\n  ImageThumbnail,\n  ImageThumbnailDivinerConfig,\n  ImageThumbnailDivinerConfigSchema,\n  ImageThumbnailDivinerParams,\n  ImageThumbnailResult,\n  ImageThumbnailResultIndexSchema,\n  ImageThumbnailResultInfo,\n  ImageThumbnailSchema,\n  isImageThumbnail,\n  isImageThumbnailDivinerQuery,\n  isImageThumbnailResult,\n} from '@xyo-network/image-thumbnail-payload-plugin'\nimport { isModuleState, ModuleState, ModuleStateSchema, StateDictionary } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\nimport { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'\n\nexport type ImageThumbnailDivinerState = StateDictionary & {\n  offset: number\n}\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore' | 'thumbnailStore'\n\ntype ConfigStore = Extract<keyof ImageThumbnailDivinerConfig, ConfigStoreKey>\n\n/**\n * The fields that will need to be indexed on in the underlying store\n */\ntype QueryableImageThumbnailResultProperties = Extract<keyof ImageThumbnailResultInfo, 'status' | 'success' | 'timestamp' | 'url'>\n\n/**\n * The query that will be used to retrieve the results from the underlying store\n */\ntype ImageThumbnailResultQuery = PayloadDivinerQueryPayload & { schemas: [ImageThumbnailSchema] } & Pick<\n    ImageThumbnailResult,\n    QueryableImageThumbnailResultProperties\n  >\n\nconst moduleName = 'ImageThumbnailDiviner'\n\nexport class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {\n  static override configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema]\n\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_0000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works in the background to populate index for the Diviner\n   * @returns\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = (await this.retrieveState()) ?? { offset: 0 }\n    const { offset } = lastState\n    // Get next batch of results\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit,\n        offset,\n        order: 'asc',\n        payload_schemas: [ImageThumbnailSchema, TimestampSchema],\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return\n    // Find all the indexable hashes in this batch\n    type IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>\n    const indexableHashes: IndexableHashes[] = (\n      await Promise.all(\n        batch.filter(isBoundWitness).map(async (bw) => {\n          const imageThumbnailIndexes = bw.payload_schemas\n            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))\n            .filter(exists)\n          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)\n          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined\n          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)\n          const timestampHash = bw.payload_hashes?.[timestampIndex]\n          const boundWitnessHash = await PayloadHasher.hashAsync(bw)\n          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)\n        }),\n      )\n    )\n      .flat()\n      .filter(exists)\n    const archivist = await this.getArchivistForStore('thumbnailStore')\n    // Find all the indexable data associated with the indexable hashes\n    type IndexableData = Readonly<\n      [\n        boundWitnessHash: string,\n        imageThumbnailHash: string,\n        imageThumbnailPayload: ImageThumbnail,\n        timestampHash: string,\n        timestampPayload: TimeStamp,\n      ]\n    >\n    const indexableData: IndexableData[] = (\n      await Promise.all(\n        indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {\n          const results = await archivist.get([imageThumbnailHash, timestampHash])\n          const imageThumbnailPayload = results.find(isImageThumbnail)\n          const timestampPayload = results.find(isTimestamp)\n          if (!imageThumbnailPayload || !timestampPayload) {\n            console.log('Could not find payload from hash to index')\n            return undefined\n          }\n          const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload)\n          const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload)\n          if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash) return undefined\n          return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const\n        }),\n      )\n    ).filter(exists)\n    // Build index results from the indexable data\n    const indexes: ImageThumbnailResult[] = indexableData.map(\n      ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {\n        const { sourceUrl: url } = thumbnailPayload\n        const { timestamp } = timestampPayload\n        const status = thumbnailPayload.http?.status\n        //call anything with a thumbnail url a success\n        const success = !!thumbnailPayload.url\n        const sources = [boundWitnessHash, thumbnailHash, timestampHash]\n        const fields = { sources, status, success, timestamp, url }\n        const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema }).fields(fields).build()\n        return result\n      },\n    )\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    const nextOffset = offset + batch.length\n    const currentState = { ...lastState, offset: nextOffset }\n    await this.commitState(currentState)\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   */\n  protected async commitState(state: ImageThumbnailDivinerState) {\n    const archivist = await this.getArchivistForStore('stateStore')\n    const payload = new PayloadBuilder<ModuleState<ImageThumbnailDivinerState>>({ schema: ModuleStateSchema }).fields({ state }).build()\n    const [bw] = await new BoundWitnessBuilder().payloads([payload]).witness(this.account).build()\n    await archivist.insert([bw, payload])\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResult[]> {\n    const urls = payloads.filter(isImageThumbnailDivinerQuery)\n    const diviner = await this.getPayloadDivinerForStore('indexStore')\n    const results = (\n      await Promise.all(\n        urls.map(async (payload) => {\n          const { limit: payloadLimit, offset: payloadOffset, order: payloadOrder, status: payloadStatus, success: payloadSuccess, url } = payload\n          const limit = payloadLimit ?? 1\n          const order = payloadOrder ?? 'desc'\n          const offset = payloadOffset ?? 0\n          const success = payloadSuccess\n          const status = payloadStatus\n          const fields: Partial<ImageThumbnailResultQuery> = { ...{ limit, offset, order, status, success, url } }\n          const query = new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema }).fields(fields).build()\n          return await diviner.divine([query])\n        }),\n      )\n    )\n      .flat()\n      .filter(isImageThumbnailResult)\n    return results\n  }\n\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ImageThumbnailDivinerState | undefined> {\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState)\n      if (payload) {\n        return payload.state as ImageThumbnailDivinerState\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,oBAAuB;AACvB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,kBAA8B;AAC9B,wCAAgF;AAChF,2BAAoC;AACpC,mCAAsE;AACtE,6BAA+B;AAC/B,4CAYO;AACP,0BAA+E;AAC/E,6BAA+B;AAE/B,+BAAwD;AAuBxD,IAAM,aAAa;AAEZ,IAAM,wBAAN,cAA+G,wCAAyB;AAAA,EAC7I,OAAgB,gBAAgB,CAAC,yEAAmC,wCAAmB;AAAA,EAE/E;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAa,MAAM,KAAK,cAAc,KAAM,EAAE,QAAQ,EAAE;AAC9D,UAAM,EAAE,OAAO,IAAI;AAEnB,UAAM,sBAAsB,MAAM,KAAK,+BAA+B,gBAAgB;AACtF,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,OAAO,KAAK;AAAA,MACZ;AAAA,MACA,OAAO;AAAA,MACP,iBAAiB,CAAC,4DAAsB,wCAAe;AAAA,IACzD,CAAC,EACA,MAAM;AACT,UAAM,QAAQ,MAAM,oBAAoB,OAAO,CAAC,KAAK,CAAC;AACtD,QAAI,MAAM,WAAW;AAAG;AAGxB,UAAM,mBACJ,MAAM,QAAQ;AAAA,MACZ,MAAM,OAAO,wCAAc,EAAE,IAAI,OAAO,OAAO;AAzFvD;AA0FU,cAAM,yBAAwB,QAAG,oBAAH,mBAC1B,IAAI,CAAC,QAAQ,UAAW,WAAW,6DAAuB,QAAQ,QACnE,OAAO;AACV,cAAM,kBAAiB,QAAG,oBAAH,mBAAoB,UAAU,CAAC,WAAW,WAAW;AAC5E,YAAI,CAAC,sBAAsB,UAAU,mBAAmB;AAAI,iBAAO;AACnE,cAAM,kBAAkB,GAAG,eAAe,IAAI,CAAC,MAAM,UAAW,sBAAsB,SAAS,KAAK,IAAI,OAAO,MAAU,EAAE,OAAO,oBAAM;AACxI,cAAM,iBAAgB,QAAG,mBAAH,mBAAoB;AAC1C,cAAM,mBAAmB,MAAM,0BAAc,UAAU,EAAE;AACzD,eAAO,gBAAgB,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,oBAAoB,aAAa,CAAU;AAAA,MACnH,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,oBAAM;AAChB,UAAM,YAAY,MAAM,KAAK,qBAAqB,gBAAgB;AAWlE,UAAM,iBACJ,MAAM,QAAQ;AAAA,MACZ,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,oBAAoB,aAAa,MAAM;AACnF,cAAM,UAAU,MAAM,UAAU,IAAI,CAAC,oBAAoB,aAAa,CAAC;AACvE,cAAM,wBAAwB,QAAQ,KAAK,sDAAgB;AAC3D,cAAM,mBAAmB,QAAQ,KAAK,oCAAW;AACjD,YAAI,CAAC,yBAAyB,CAAC,kBAAkB;AAC/C,kBAAQ,IAAI,2CAA2C;AACvD,iBAAO;AAAA,QACT;AACA,cAAM,+BAA+B,MAAM,0BAAc,UAAU,qBAAqB;AACxF,cAAM,0BAA0B,MAAM,0BAAc,UAAU,gBAAgB;AAC9E,YAAI,uBAAuB,gCAAgC,kBAAkB;AAAyB,iBAAO;AAC7G,eAAO,CAAC,kBAAkB,oBAAoB,uBAAuB,eAAe,gBAAgB;AAAA,MACtG,CAAC;AAAA,IACH,GACA,OAAO,oBAAM;AAEf,UAAM,UAAkC,cAAc;AAAA,MACpD,CAAC,CAAC,kBAAkB,eAAe,kBAAkB,eAAe,gBAAgB,MAAM;AAtIhG;AAuIQ,cAAM,EAAE,WAAW,IAAI,IAAI;AAC3B,cAAM,EAAE,UAAU,IAAI;AACtB,cAAM,UAAS,sBAAiB,SAAjB,mBAAuB;AAEtC,cAAM,UAAU,CAAC,CAAC,iBAAiB;AACnC,cAAM,UAAU,CAAC,kBAAkB,eAAe,aAAa;AAC/D,cAAM,SAAS,EAAE,SAAS,QAAQ,SAAS,WAAW,IAAI;AAC1D,cAAM,SAAS,IAAI,sCAAqC,EAAE,QAAQ,sEAAgC,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AAC1H,eAAO;AAAA,MACT;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,UAAM,aAAa,SAAS,MAAM;AAClC,UAAM,eAAe,EAAE,GAAG,WAAW,QAAQ,WAAW;AACxD,UAAM,KAAK,YAAY,YAAY;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAgB,YAAY,OAAmC;AAC7D,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,UAAU,IAAI,sCAAwD,EAAE,QAAQ,sCAAkB,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM;AACnI,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC7F,UAAM,UAAU,OAAO,CAAC,IAAI,OAAO,CAAC;AAAA,EACtC;AAAA,EAEA,MAAyB,cAAc,WAAsB,CAAC,GAAoC;AAChG,UAAM,OAAO,SAAS,OAAO,kEAA4B;AACzD,UAAM,UAAU,MAAM,KAAK,0BAA0B,YAAY;AACjE,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,KAAK,IAAI,OAAO,YAAY;AAC1B,cAAM,EAAE,OAAO,cAAc,QAAQ,eAAe,OAAO,cAAc,QAAQ,eAAe,SAAS,gBAAgB,IAAI,IAAI;AACjI,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,SAAS,iBAAiB;AAChC,cAAM,UAAU;AAChB,cAAM,SAAS;AACf,cAAM,SAA6C,EAAE,GAAG,EAAE,OAAO,QAAQ,OAAO,QAAQ,SAAS,IAAI,EAAE;AACvG,cAAM,QAAQ,IAAI,sCAA0C,EAAE,QAAQ,uDAA0B,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AACxH,eAAO,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AAAA,MACrC,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,4DAAsB;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,MAAgB,qBAAqB,OAAoB;AA/L3D;AAgMI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA,EAEA,MAAgB,+BAA+B,OAAoB;AArMrE;AAsMI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA,EAEA,MAAgB,0BAA0B,OAAoB;AA3MhE;AA4MI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAiE;AAC/E,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AA1OzB;AA0O6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,wCAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAa;AAChE,UAAI,SAAS;AACX,eAAO,QAAQ;AAAA,MACjB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA,EAEQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
+{"version":3,"sources":["../../src/index.ts","../../src/Diviner/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistInstance } from '@xyo-network/archivist-model'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { PayloadHasher } from '@xyo-network/core'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport { DivinerConfigSchema } from '@xyo-network/diviner-model'\nimport { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport {\n  ImageThumbnail,\n  ImageThumbnailDivinerConfig,\n  ImageThumbnailDivinerConfigSchema,\n  ImageThumbnailDivinerParams,\n  ImageThumbnailResult,\n  ImageThumbnailResultIndexSchema,\n  ImageThumbnailResultInfo,\n  ImageThumbnailSchema,\n  isImageThumbnail,\n  isImageThumbnailDivinerQuery,\n  isImageThumbnailResult,\n} from '@xyo-network/image-thumbnail-payload-plugin'\nimport { isModuleState, ModuleState, ModuleStateSchema, StateDictionary } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\nimport { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'\n\nexport type ImageThumbnailDivinerState = StateDictionary & {\n  offset: number\n}\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore' | 'thumbnailStore'\n\ntype ConfigStore = Extract<keyof ImageThumbnailDivinerConfig, ConfigStoreKey>\n\n/**\n * The fields that will need to be indexed on in the underlying store\n */\ntype QueryableImageThumbnailResultProperties = Extract<keyof ImageThumbnailResultInfo, 'status' | 'success' | 'timestamp' | 'url'>\n\n/**\n * The query that will be used to retrieve the results from the underlying store\n */\ntype ImageThumbnailResultQuery = PayloadDivinerQueryPayload & { schemas: [ImageThumbnailSchema] } & Pick<\n    ImageThumbnailResult,\n    QueryableImageThumbnailResultProperties\n  >\n\ntype IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>\n\nconst moduleName = 'ImageThumbnailDiviner'\n\nexport class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {\n  static override configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema]\n\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_0000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  private static async indexableData(indexableHashes: IndexableHashes[], archivist: ArchivistInstance) {\n    // Find all the indexable data associated with the indexable hashes\n    type IndexableData = Readonly<\n      [\n        boundWitnessHash: string,\n        imageThumbnailHash: string,\n        imageThumbnailPayload: ImageThumbnail,\n        timestampHash: string,\n        timestampPayload: TimeStamp,\n      ]\n    >\n    const indexableData: IndexableData[] = (\n      await Promise.all(\n        indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {\n          const results = await archivist.get([imageThumbnailHash, timestampHash])\n          const imageThumbnailPayload = results.find(isImageThumbnail)\n          const timestampPayload = results.find(isTimestamp)\n          if (!imageThumbnailPayload || !timestampPayload) {\n            console.log('Could not find payload from hash to index')\n            return undefined\n          }\n          const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload)\n          const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload)\n          if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash) return undefined\n          return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const\n        }),\n      )\n    ).filter(exists)\n    return indexableData\n  }\n\n  private static async indexableHashes(batch: Payload[]) {\n    // Find all the indexable hashes in this batch\n    const indexableHashes: IndexableHashes[] = (\n      await Promise.all(\n        batch.filter(isBoundWitness).map(async (bw) => {\n          const imageThumbnailIndexes = bw.payload_schemas\n            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))\n            .filter(exists)\n          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)\n          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined\n          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)\n          const timestampHash = bw.payload_hashes?.[timestampIndex]\n          const boundWitnessHash = await PayloadHasher.hashAsync(bw)\n          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)\n        }),\n      )\n    )\n      .flat()\n      .filter(exists)\n    return indexableHashes\n  }\n\n  /**\n   * Works in the background to populate index for the Diviner\n   * @returns\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = (await this.retrieveState()) ?? { offset: 0 }\n    const { offset } = lastState\n    // Get next batch of results\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit,\n        offset,\n        order: 'asc',\n        payload_schemas: [ImageThumbnailSchema, TimestampSchema],\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return\n    const indexableHashes = await ImageThumbnailDiviner.indexableHashes(batch)\n    const archivist = await this.getArchivistForStore('thumbnailStore')\n    const indexableData = await ImageThumbnailDiviner.indexableData(indexableHashes, archivist)\n    // Build index results from the indexable data\n    const indexes: ImageThumbnailResult[] = indexableData.map(\n      ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {\n        const { sourceUrl: url } = thumbnailPayload\n        const { timestamp } = timestampPayload\n        const status = thumbnailPayload.http?.status\n        //call anything with a thumbnail url a success\n        const success = !!thumbnailPayload.url\n        const sources = [boundWitnessHash, thumbnailHash, timestampHash]\n        const fields = { sources, status, success, timestamp, url }\n        const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema }).fields(fields).build()\n        return result\n      },\n    )\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    const nextOffset = offset + batch.length\n    const currentState = { ...lastState, offset: nextOffset }\n    await this.commitState(currentState)\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   */\n  protected async commitState(state: ImageThumbnailDivinerState) {\n    const archivist = await this.getArchivistForStore('stateStore')\n    const payload = new PayloadBuilder<ModuleState<ImageThumbnailDivinerState>>({ schema: ModuleStateSchema }).fields({ state }).build()\n    const [bw] = await new BoundWitnessBuilder().payloads([payload]).witness(this.account).build()\n    await archivist.insert([bw, payload])\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResult[]> {\n    const urls = payloads.filter(isImageThumbnailDivinerQuery)\n    const diviner = await this.getPayloadDivinerForStore('indexStore')\n    const results = (\n      await Promise.all(\n        urls.map(async (payload) => {\n          const { limit: payloadLimit, offset: payloadOffset, order: payloadOrder, status: payloadStatus, success: payloadSuccess, url } = payload\n          const limit = payloadLimit ?? 1\n          const order = payloadOrder ?? 'desc'\n          const offset = payloadOffset ?? 0\n          const success = payloadSuccess\n          const status = payloadStatus\n          const fields: Partial<ImageThumbnailResultQuery> = { ...{ limit, offset, order, status, success, url } }\n          const query = new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema }).fields(fields).build()\n          return await diviner.divine([query])\n        }),\n      )\n    )\n      .flat()\n      .filter(isImageThumbnailResult)\n    return results\n  }\n\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ImageThumbnailDivinerState | undefined> {\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState)\n      if (payload) {\n        return payload.state as ImageThumbnailDivinerState\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,oBAAuB;AACvB,8BAAgC;AAEhC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,kBAA8B;AAC9B,wCAAgF;AAChF,2BAAoC;AACpC,mCAAsE;AACtE,6BAA+B;AAC/B,4CAYO;AACP,0BAA+E;AAC/E,6BAA+B;AAE/B,+BAAwD;AAyBxD,IAAM,aAAa;AAEZ,IAAM,wBAAN,MAAM,+BAAyG,wCAAyB;AAAA,EAC7I,OAAgB,gBAAgB,CAAC,yEAAmC,wCAAmB;AAAA,EAE/E;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA,EAEA,aAAqB,cAAc,iBAAoC,WAA8B;AAWnG,UAAM,iBACJ,MAAM,QAAQ;AAAA,MACZ,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,oBAAoB,aAAa,MAAM;AACnF,cAAM,UAAU,MAAM,UAAU,IAAI,CAAC,oBAAoB,aAAa,CAAC;AACvE,cAAM,wBAAwB,QAAQ,KAAK,sDAAgB;AAC3D,cAAM,mBAAmB,QAAQ,KAAK,oCAAW;AACjD,YAAI,CAAC,yBAAyB,CAAC,kBAAkB;AAC/C,kBAAQ,IAAI,2CAA2C;AACvD,iBAAO;AAAA,QACT;AACA,cAAM,+BAA+B,MAAM,0BAAc,UAAU,qBAAqB;AACxF,cAAM,0BAA0B,MAAM,0BAAc,UAAU,gBAAgB;AAC9E,YAAI,uBAAuB,gCAAgC,kBAAkB;AAAyB,iBAAO;AAC7G,eAAO,CAAC,kBAAkB,oBAAoB,uBAAuB,eAAe,gBAAgB;AAAA,MACtG,CAAC;AAAA,IACH,GACA,OAAO,oBAAM;AACf,WAAO;AAAA,EACT;AAAA,EAEA,aAAqB,gBAAgB,OAAkB;AAErD,UAAM,mBACJ,MAAM,QAAQ;AAAA,MACZ,MAAM,OAAO,wCAAc,EAAE,IAAI,OAAO,OAAO;AAvGvD;AAwGU,cAAM,yBAAwB,QAAG,oBAAH,mBAC1B,IAAI,CAAC,QAAQ,UAAW,WAAW,6DAAuB,QAAQ,QACnE,OAAO;AACV,cAAM,kBAAiB,QAAG,oBAAH,mBAAoB,UAAU,CAAC,WAAW,WAAW;AAC5E,YAAI,CAAC,sBAAsB,UAAU,mBAAmB;AAAI,iBAAO;AACnE,cAAM,kBAAkB,GAAG,eAAe,IAAI,CAAC,MAAM,UAAW,sBAAsB,SAAS,KAAK,IAAI,OAAO,MAAU,EAAE,OAAO,oBAAM;AACxI,cAAM,iBAAgB,QAAG,mBAAH,mBAAoB;AAC1C,cAAM,mBAAmB,MAAM,0BAAc,UAAU,EAAE;AACzD,eAAO,gBAAgB,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,oBAAoB,aAAa,CAAU;AAAA,MACnH,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,oBAAM;AAChB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAa,MAAM,KAAK,cAAc,KAAM,EAAE,QAAQ,EAAE;AAC9D,UAAM,EAAE,OAAO,IAAI;AAEnB,UAAM,sBAAsB,MAAM,KAAK,+BAA+B,gBAAgB;AACtF,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,OAAO,KAAK;AAAA,MACZ;AAAA,MACA,OAAO;AAAA,MACP,iBAAiB,CAAC,4DAAsB,wCAAe;AAAA,IACzD,CAAC,EACA,MAAM;AACT,UAAM,QAAQ,MAAM,oBAAoB,OAAO,CAAC,KAAK,CAAC;AACtD,QAAI,MAAM,WAAW;AAAG;AACxB,UAAM,kBAAkB,MAAM,uBAAsB,gBAAgB,KAAK;AACzE,UAAM,YAAY,MAAM,KAAK,qBAAqB,gBAAgB;AAClE,UAAM,gBAAgB,MAAM,uBAAsB,cAAc,iBAAiB,SAAS;AAE1F,UAAM,UAAkC,cAAc;AAAA,MACpD,CAAC,CAAC,kBAAkB,eAAe,kBAAkB,eAAe,gBAAgB,MAAM;AAlJhG;AAmJQ,cAAM,EAAE,WAAW,IAAI,IAAI;AAC3B,cAAM,EAAE,UAAU,IAAI;AACtB,cAAM,UAAS,sBAAiB,SAAjB,mBAAuB;AAEtC,cAAM,UAAU,CAAC,CAAC,iBAAiB;AACnC,cAAM,UAAU,CAAC,kBAAkB,eAAe,aAAa;AAC/D,cAAM,SAAS,EAAE,SAAS,QAAQ,SAAS,WAAW,IAAI;AAC1D,cAAM,SAAS,IAAI,sCAAqC,EAAE,QAAQ,sEAAgC,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AAC1H,eAAO;AAAA,MACT;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,UAAM,aAAa,SAAS,MAAM;AAClC,UAAM,eAAe,EAAE,GAAG,WAAW,QAAQ,WAAW;AACxD,UAAM,KAAK,YAAY,YAAY;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAgB,YAAY,OAAmC;AAC7D,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,UAAU,IAAI,sCAAwD,EAAE,QAAQ,sCAAkB,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM;AACnI,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC7F,UAAM,UAAU,OAAO,CAAC,IAAI,OAAO,CAAC;AAAA,EACtC;AAAA,EAEA,MAAyB,cAAc,WAAsB,CAAC,GAAoC;AAChG,UAAM,OAAO,SAAS,OAAO,kEAA4B;AACzD,UAAM,UAAU,MAAM,KAAK,0BAA0B,YAAY;AACjE,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,KAAK,IAAI,OAAO,YAAY;AAC1B,cAAM,EAAE,OAAO,cAAc,QAAQ,eAAe,OAAO,cAAc,QAAQ,eAAe,SAAS,gBAAgB,IAAI,IAAI;AACjI,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,SAAS,iBAAiB;AAChC,cAAM,UAAU;AAChB,cAAM,SAAS;AACf,cAAM,SAA6C,EAAE,GAAG,EAAE,OAAO,QAAQ,OAAO,QAAQ,SAAS,IAAI,EAAE;AACvG,cAAM,QAAQ,IAAI,sCAA0C,EAAE,QAAQ,uDAA0B,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AACxH,eAAO,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AAAA,MACrC,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,4DAAsB;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,MAAgB,qBAAqB,OAAoB;AA3M3D;AA4MI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA,EAEA,MAAgB,+BAA+B,OAAoB;AAjNrE;AAkNI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA,EAEA,MAAgB,0BAA0B,OAAoB;AAvNhE;AAwNI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAiE;AAC/E,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AAtPzB;AAsP6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,wCAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAa;AAChE,UAAI,SAAS;AACX,eAAO,QAAQ;AAAA,MACjB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA,EAEQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}

package/dist/node/index.mjs

@@ -22,7 +22,7 @@ import { isModuleState, ModuleStateSchema } from "@xyo-network/module-model";
 import { PayloadBuilder } from "@xyo-network/payload-builder";
 import { isTimestamp, TimestampSchema } from "@xyo-network/witness-timestamp";
 var moduleName = "ImageThumbnailDiviner";
-var ImageThumbnailDiviner = class extends AbstractDiviner {
+var ImageThumbnailDiviner = class _ImageThumbnailDiviner extends AbstractDiviner {
   static configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema];
   _pollId;
   get payloadDivinerLimit() {
@@ -31,6 +31,41 @@ var ImageThumbnailDiviner = class extends AbstractDiviner {
   get pollFrequency() {
     return this.config.pollFrequency ?? 1e4;
   }
+  static async indexableData(indexableHashes, archivist) {
+    const indexableData = (await Promise.all(
+      indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {
+        const results = await archivist.get([imageThumbnailHash, timestampHash]);
+        const imageThumbnailPayload = results.find(isImageThumbnail);
+        const timestampPayload = results.find(isTimestamp);
+        if (!imageThumbnailPayload || !timestampPayload) {
+          console.log("Could not find payload from hash to index");
+          return void 0;
+        }
+        const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload);
+        const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload);
+        if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash)
+          return void 0;
+        return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload];
+      })
+    )).filter(exists);
+    return indexableData;
+  }
+  static async indexableHashes(batch) {
+    const indexableHashes = (await Promise.all(
+      batch.filter(isBoundWitness).map(async (bw) => {
+        var _a, _b, _c;
+        const imageThumbnailIndexes = (_a = bw.payload_schemas) == null ? void 0 : _a.map((schema, index) => schema === ImageThumbnailSchema ? index : void 0).filter(exists);
+        const timestampIndex = (_b = bw.payload_schemas) == null ? void 0 : _b.findIndex((schema) => schema === TimestampSchema);
+        if (!imageThumbnailIndexes.length || timestampIndex === -1)
+          return void 0;
+        const imageThumbnails = bw.payload_hashes.map((hash, index) => imageThumbnailIndexes.includes(index) ? hash : void 0).filter(exists);
+        const timestampHash = (_c = bw.payload_hashes) == null ? void 0 : _c[timestampIndex];
+        const boundWitnessHash = await PayloadHasher.hashAsync(bw);
+        return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash]);
+      })
+    )).flat().filter(exists);
+    return indexableHashes;
+  }
   /**
    * Works in the background to populate index for the Diviner
    * @returns
@@ -48,36 +83,9 @@ var ImageThumbnailDiviner = class extends AbstractDiviner {
     const batch = await boundWitnessDiviner.divine([query]);
     if (batch.length === 0)
       return;
-    const indexableHashes = (await Promise.all(
-      batch.filter(isBoundWitness).map(async (bw) => {
-        var _a, _b, _c;
-        const imageThumbnailIndexes = (_a = bw.payload_schemas) == null ? void 0 : _a.map((schema, index) => schema === ImageThumbnailSchema ? index : void 0).filter(exists);
-        const timestampIndex = (_b = bw.payload_schemas) == null ? void 0 : _b.findIndex((schema) => schema === TimestampSchema);
-        if (!imageThumbnailIndexes.length || timestampIndex === -1)
-          return void 0;
-        const imageThumbnails = bw.payload_hashes.map((hash, index) => imageThumbnailIndexes.includes(index) ? hash : void 0).filter(exists);
-        const timestampHash = (_c = bw.payload_hashes) == null ? void 0 : _c[timestampIndex];
-        const boundWitnessHash = await PayloadHasher.hashAsync(bw);
-        return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash]);
-      })
-    )).flat().filter(exists);
+    const indexableHashes = await _ImageThumbnailDiviner.indexableHashes(batch);
     const archivist = await this.getArchivistForStore("thumbnailStore");
-    const indexableData = (await Promise.all(
-      indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {
-        const results = await archivist.get([imageThumbnailHash, timestampHash]);
-        const imageThumbnailPayload = results.find(isImageThumbnail);
-        const timestampPayload = results.find(isTimestamp);
-        if (!imageThumbnailPayload || !timestampPayload) {
-          console.log("Could not find payload from hash to index");
-          return void 0;
-        }
-        const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload);
-        const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload);
-        if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash)
-          return void 0;
-        return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload];
-      })
-    )).filter(exists);
+    const indexableData = await _ImageThumbnailDiviner.indexableData(indexableHashes, archivist);
     const indexes = indexableData.map(
       ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {
         var _a;

package/dist/node/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { PayloadHasher } from '@xyo-network/core'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport { DivinerConfigSchema } from '@xyo-network/diviner-model'\nimport { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport {\n  ImageThumbnail,\n  ImageThumbnailDivinerConfig,\n  ImageThumbnailDivinerConfigSchema,\n  ImageThumbnailDivinerParams,\n  ImageThumbnailResult,\n  ImageThumbnailResultIndexSchema,\n  ImageThumbnailResultInfo,\n  ImageThumbnailSchema,\n  isImageThumbnail,\n  isImageThumbnailDivinerQuery,\n  isImageThumbnailResult,\n} from '@xyo-network/image-thumbnail-payload-plugin'\nimport { isModuleState, ModuleState, ModuleStateSchema, StateDictionary } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\nimport { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'\n\nexport type ImageThumbnailDivinerState = StateDictionary & {\n  offset: number\n}\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore' | 'thumbnailStore'\n\ntype ConfigStore = Extract<keyof ImageThumbnailDivinerConfig, ConfigStoreKey>\n\n/**\n * The fields that will need to be indexed on in the underlying store\n */\ntype QueryableImageThumbnailResultProperties = Extract<keyof ImageThumbnailResultInfo, 'status' | 'success' | 'timestamp' | 'url'>\n\n/**\n * The query that will be used to retrieve the results from the underlying store\n */\ntype ImageThumbnailResultQuery = PayloadDivinerQueryPayload & { schemas: [ImageThumbnailSchema] } & Pick<\n    ImageThumbnailResult,\n    QueryableImageThumbnailResultProperties\n  >\n\nconst moduleName = 'ImageThumbnailDiviner'\n\nexport class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {\n  static override configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema]\n\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_0000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works in the background to populate index for the Diviner\n   * @returns\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = (await this.retrieveState()) ?? { offset: 0 }\n    const { offset } = lastState\n    // Get next batch of results\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit,\n        offset,\n        order: 'asc',\n        payload_schemas: [ImageThumbnailSchema, TimestampSchema],\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return\n    // Find all the indexable hashes in this batch\n    type IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>\n    const indexableHashes: IndexableHashes[] = (\n      await Promise.all(\n        batch.filter(isBoundWitness).map(async (bw) => {\n          const imageThumbnailIndexes = bw.payload_schemas\n            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))\n            .filter(exists)\n          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)\n          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined\n          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)\n          const timestampHash = bw.payload_hashes?.[timestampIndex]\n          const boundWitnessHash = await PayloadHasher.hashAsync(bw)\n          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)\n        }),\n      )\n    )\n      .flat()\n      .filter(exists)\n    const archivist = await this.getArchivistForStore('thumbnailStore')\n    // Find all the indexable data associated with the indexable hashes\n    type IndexableData = Readonly<\n      [\n        boundWitnessHash: string,\n        imageThumbnailHash: string,\n        imageThumbnailPayload: ImageThumbnail,\n        timestampHash: string,\n        timestampPayload: TimeStamp,\n      ]\n    >\n    const indexableData: IndexableData[] = (\n      await Promise.all(\n        indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {\n          const results = await archivist.get([imageThumbnailHash, timestampHash])\n          const imageThumbnailPayload = results.find(isImageThumbnail)\n          const timestampPayload = results.find(isTimestamp)\n          if (!imageThumbnailPayload || !timestampPayload) {\n            console.log('Could not find payload from hash to index')\n            return undefined\n          }\n          const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload)\n          const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload)\n          if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash) return undefined\n          return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const\n        }),\n      )\n    ).filter(exists)\n    // Build index results from the indexable data\n    const indexes: ImageThumbnailResult[] = indexableData.map(\n      ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {\n        const { sourceUrl: url } = thumbnailPayload\n        const { timestamp } = timestampPayload\n        const status = thumbnailPayload.http?.status\n        //call anything with a thumbnail url a success\n        const success = !!thumbnailPayload.url\n        const sources = [boundWitnessHash, thumbnailHash, timestampHash]\n        const fields = { sources, status, success, timestamp, url }\n        const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema }).fields(fields).build()\n        return result\n      },\n    )\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    const nextOffset = offset + batch.length\n    const currentState = { ...lastState, offset: nextOffset }\n    await this.commitState(currentState)\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   */\n  protected async commitState(state: ImageThumbnailDivinerState) {\n    const archivist = await this.getArchivistForStore('stateStore')\n    const payload = new PayloadBuilder<ModuleState<ImageThumbnailDivinerState>>({ schema: ModuleStateSchema }).fields({ state }).build()\n    const [bw] = await new BoundWitnessBuilder().payloads([payload]).witness(this.account).build()\n    await archivist.insert([bw, payload])\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResult[]> {\n    const urls = payloads.filter(isImageThumbnailDivinerQuery)\n    const diviner = await this.getPayloadDivinerForStore('indexStore')\n    const results = (\n      await Promise.all(\n        urls.map(async (payload) => {\n          const { limit: payloadLimit, offset: payloadOffset, order: payloadOrder, status: payloadStatus, success: payloadSuccess, url } = payload\n          const limit = payloadLimit ?? 1\n          const order = payloadOrder ?? 'desc'\n          const offset = payloadOffset ?? 0\n          const success = payloadSuccess\n          const status = payloadStatus\n          const fields: Partial<ImageThumbnailResultQuery> = { ...{ limit, offset, order, status, success, url } }\n          const query = new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema }).fields(fields).build()\n          return await diviner.divine([query])\n        }),\n      )\n    )\n      .flat()\n      .filter(isImageThumbnailResult)\n    return results\n  }\n\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ImageThumbnailDivinerState | undefined> {\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState)\n      if (payload) {\n        return payload.state as ImageThumbnailDivinerState\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,cAAc;AACvB,SAAS,uBAAuB;AAChC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAAS,qBAAqB;AAC9B,SAA0C,sCAAsC;AAChF,SAAS,2BAA2B;AACpC,SAAqC,iCAAiC;AACtE,SAAS,sBAAsB;AAC/B;AAAA,EAGE;AAAA,EAGA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,eAA4B,yBAA0C;AAC/E,SAAS,sBAAsB;AAE/B,SAAS,aAAwB,uBAAuB;AAuBxD,IAAM,aAAa;AAEZ,IAAM,wBAAN,cAA+G,gBAAyB;AAAA,EAC7I,OAAgB,gBAAgB,CAAC,mCAAmC,mBAAmB;AAAA,EAE/E;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAa,MAAM,KAAK,cAAc,KAAM,EAAE,QAAQ,EAAE;AAC9D,UAAM,EAAE,OAAO,IAAI;AAEnB,UAAM,sBAAsB,MAAM,KAAK,+BAA+B,gBAAgB;AACtF,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,OAAO,KAAK;AAAA,MACZ;AAAA,MACA,OAAO;AAAA,MACP,iBAAiB,CAAC,sBAAsB,eAAe;AAAA,IACzD,CAAC,EACA,MAAM;AACT,UAAM,QAAQ,MAAM,oBAAoB,OAAO,CAAC,KAAK,CAAC;AACtD,QAAI,MAAM,WAAW;AAAG;AAGxB,UAAM,mBACJ,MAAM,QAAQ;AAAA,MACZ,MAAM,OAAO,cAAc,EAAE,IAAI,OAAO,OAAO;AAzFvD;AA0FU,cAAM,yBAAwB,QAAG,oBAAH,mBAC1B,IAAI,CAAC,QAAQ,UAAW,WAAW,uBAAuB,QAAQ,QACnE,OAAO;AACV,cAAM,kBAAiB,QAAG,oBAAH,mBAAoB,UAAU,CAAC,WAAW,WAAW;AAC5E,YAAI,CAAC,sBAAsB,UAAU,mBAAmB;AAAI,iBAAO;AACnE,cAAM,kBAAkB,GAAG,eAAe,IAAI,CAAC,MAAM,UAAW,sBAAsB,SAAS,KAAK,IAAI,OAAO,MAAU,EAAE,OAAO,MAAM;AACxI,cAAM,iBAAgB,QAAG,mBAAH,mBAAoB;AAC1C,cAAM,mBAAmB,MAAM,cAAc,UAAU,EAAE;AACzD,eAAO,gBAAgB,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,oBAAoB,aAAa,CAAU;AAAA,MACnH,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,MAAM;AAChB,UAAM,YAAY,MAAM,KAAK,qBAAqB,gBAAgB;AAWlE,UAAM,iBACJ,MAAM,QAAQ;AAAA,MACZ,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,oBAAoB,aAAa,MAAM;AACnF,cAAM,UAAU,MAAM,UAAU,IAAI,CAAC,oBAAoB,aAAa,CAAC;AACvE,cAAM,wBAAwB,QAAQ,KAAK,gBAAgB;AAC3D,cAAM,mBAAmB,QAAQ,KAAK,WAAW;AACjD,YAAI,CAAC,yBAAyB,CAAC,kBAAkB;AAC/C,kBAAQ,IAAI,2CAA2C;AACvD,iBAAO;AAAA,QACT;AACA,cAAM,+BAA+B,MAAM,cAAc,UAAU,qBAAqB;AACxF,cAAM,0BAA0B,MAAM,cAAc,UAAU,gBAAgB;AAC9E,YAAI,uBAAuB,gCAAgC,kBAAkB;AAAyB,iBAAO;AAC7G,eAAO,CAAC,kBAAkB,oBAAoB,uBAAuB,eAAe,gBAAgB;AAAA,MACtG,CAAC;AAAA,IACH,GACA,OAAO,MAAM;AAEf,UAAM,UAAkC,cAAc;AAAA,MACpD,CAAC,CAAC,kBAAkB,eAAe,kBAAkB,eAAe,gBAAgB,MAAM;AAtIhG;AAuIQ,cAAM,EAAE,WAAW,IAAI,IAAI;AAC3B,cAAM,EAAE,UAAU,IAAI;AACtB,cAAM,UAAS,sBAAiB,SAAjB,mBAAuB;AAEtC,cAAM,UAAU,CAAC,CAAC,iBAAiB;AACnC,cAAM,UAAU,CAAC,kBAAkB,eAAe,aAAa;AAC/D,cAAM,SAAS,EAAE,SAAS,QAAQ,SAAS,WAAW,IAAI;AAC1D,cAAM,SAAS,IAAI,eAAqC,EAAE,QAAQ,gCAAgC,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AAC1H,eAAO;AAAA,MACT;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,UAAM,aAAa,SAAS,MAAM;AAClC,UAAM,eAAe,EAAE,GAAG,WAAW,QAAQ,WAAW;AACxD,UAAM,KAAK,YAAY,YAAY;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAgB,YAAY,OAAmC;AAC7D,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,UAAU,IAAI,eAAwD,EAAE,QAAQ,kBAAkB,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM;AACnI,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC7F,UAAM,UAAU,OAAO,CAAC,IAAI,OAAO,CAAC;AAAA,EACtC;AAAA,EAEA,MAAyB,cAAc,WAAsB,CAAC,GAAoC;AAChG,UAAM,OAAO,SAAS,OAAO,4BAA4B;AACzD,UAAM,UAAU,MAAM,KAAK,0BAA0B,YAAY;AACjE,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,KAAK,IAAI,OAAO,YAAY;AAC1B,cAAM,EAAE,OAAO,cAAc,QAAQ,eAAe,OAAO,cAAc,QAAQ,eAAe,SAAS,gBAAgB,IAAI,IAAI;AACjI,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,SAAS,iBAAiB;AAChC,cAAM,UAAU;AAChB,cAAM,SAAS;AACf,cAAM,SAA6C,EAAE,GAAG,EAAE,OAAO,QAAQ,OAAO,QAAQ,SAAS,IAAI,EAAE;AACvG,cAAM,QAAQ,IAAI,eAA0C,EAAE,QAAQ,0BAA0B,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AACxH,eAAO,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AAAA,MACrC,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,sBAAsB;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,MAAgB,qBAAqB,OAAoB;AA/L3D;AAgMI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA,EAEA,MAAgB,+BAA+B,OAAoB;AArMrE;AAsMI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA,EAEA,MAAgB,0BAA0B,OAAoB;AA3MhE;AA4MI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAiE;AAC/E,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AA1OzB;AA0O6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,oBAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAa;AAChE,UAAI,SAAS;AACX,eAAO,QAAQ;AAAA,MACjB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA,EAEQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
+{"version":3,"sources":["../../src/Diviner/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistInstance } from '@xyo-network/archivist-model'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { PayloadHasher } from '@xyo-network/core'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport { DivinerConfigSchema } from '@xyo-network/diviner-model'\nimport { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport {\n  ImageThumbnail,\n  ImageThumbnailDivinerConfig,\n  ImageThumbnailDivinerConfigSchema,\n  ImageThumbnailDivinerParams,\n  ImageThumbnailResult,\n  ImageThumbnailResultIndexSchema,\n  ImageThumbnailResultInfo,\n  ImageThumbnailSchema,\n  isImageThumbnail,\n  isImageThumbnailDivinerQuery,\n  isImageThumbnailResult,\n} from '@xyo-network/image-thumbnail-payload-plugin'\nimport { isModuleState, ModuleState, ModuleStateSchema, StateDictionary } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\nimport { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'\n\nexport type ImageThumbnailDivinerState = StateDictionary & {\n  offset: number\n}\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore' | 'thumbnailStore'\n\ntype ConfigStore = Extract<keyof ImageThumbnailDivinerConfig, ConfigStoreKey>\n\n/**\n * The fields that will need to be indexed on in the underlying store\n */\ntype QueryableImageThumbnailResultProperties = Extract<keyof ImageThumbnailResultInfo, 'status' | 'success' | 'timestamp' | 'url'>\n\n/**\n * The query that will be used to retrieve the results from the underlying store\n */\ntype ImageThumbnailResultQuery = PayloadDivinerQueryPayload & { schemas: [ImageThumbnailSchema] } & Pick<\n    ImageThumbnailResult,\n    QueryableImageThumbnailResultProperties\n  >\n\ntype IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>\n\nconst moduleName = 'ImageThumbnailDiviner'\n\nexport class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {\n  static override configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema]\n\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_0000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  private static async indexableData(indexableHashes: IndexableHashes[], archivist: ArchivistInstance) {\n    // Find all the indexable data associated with the indexable hashes\n    type IndexableData = Readonly<\n      [\n        boundWitnessHash: string,\n        imageThumbnailHash: string,\n        imageThumbnailPayload: ImageThumbnail,\n        timestampHash: string,\n        timestampPayload: TimeStamp,\n      ]\n    >\n    const indexableData: IndexableData[] = (\n      await Promise.all(\n        indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {\n          const results = await archivist.get([imageThumbnailHash, timestampHash])\n          const imageThumbnailPayload = results.find(isImageThumbnail)\n          const timestampPayload = results.find(isTimestamp)\n          if (!imageThumbnailPayload || !timestampPayload) {\n            console.log('Could not find payload from hash to index')\n            return undefined\n          }\n          const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload)\n          const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload)\n          if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash) return undefined\n          return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const\n        }),\n      )\n    ).filter(exists)\n    return indexableData\n  }\n\n  private static async indexableHashes(batch: Payload[]) {\n    // Find all the indexable hashes in this batch\n    const indexableHashes: IndexableHashes[] = (\n      await Promise.all(\n        batch.filter(isBoundWitness).map(async (bw) => {\n          const imageThumbnailIndexes = bw.payload_schemas\n            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))\n            .filter(exists)\n          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)\n          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined\n          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)\n          const timestampHash = bw.payload_hashes?.[timestampIndex]\n          const boundWitnessHash = await PayloadHasher.hashAsync(bw)\n          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)\n        }),\n      )\n    )\n      .flat()\n      .filter(exists)\n    return indexableHashes\n  }\n\n  /**\n   * Works in the background to populate index for the Diviner\n   * @returns\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = (await this.retrieveState()) ?? { offset: 0 }\n    const { offset } = lastState\n    // Get next batch of results\n    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        limit: this.payloadDivinerLimit,\n        offset,\n        order: 'asc',\n        payload_schemas: [ImageThumbnailSchema, TimestampSchema],\n      })\n      .build()\n    const batch = await boundWitnessDiviner.divine([query])\n    if (batch.length === 0) return\n    const indexableHashes = await ImageThumbnailDiviner.indexableHashes(batch)\n    const archivist = await this.getArchivistForStore('thumbnailStore')\n    const indexableData = await ImageThumbnailDiviner.indexableData(indexableHashes, archivist)\n    // Build index results from the indexable data\n    const indexes: ImageThumbnailResult[] = indexableData.map(\n      ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {\n        const { sourceUrl: url } = thumbnailPayload\n        const { timestamp } = timestampPayload\n        const status = thumbnailPayload.http?.status\n        //call anything with a thumbnail url a success\n        const success = !!thumbnailPayload.url\n        const sources = [boundWitnessHash, thumbnailHash, timestampHash]\n        const fields = { sources, status, success, timestamp, url }\n        const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema }).fields(fields).build()\n        return result\n      },\n    )\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    const nextOffset = offset + batch.length\n    const currentState = { ...lastState, offset: nextOffset }\n    await this.commitState(currentState)\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   */\n  protected async commitState(state: ImageThumbnailDivinerState) {\n    const archivist = await this.getArchivistForStore('stateStore')\n    const payload = new PayloadBuilder<ModuleState<ImageThumbnailDivinerState>>({ schema: ModuleStateSchema }).fields({ state }).build()\n    const [bw] = await new BoundWitnessBuilder().payloads([payload]).witness(this.account).build()\n    await archivist.insert([bw, payload])\n  }\n\n  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResult[]> {\n    const urls = payloads.filter(isImageThumbnailDivinerQuery)\n    const diviner = await this.getPayloadDivinerForStore('indexStore')\n    const results = (\n      await Promise.all(\n        urls.map(async (payload) => {\n          const { limit: payloadLimit, offset: payloadOffset, order: payloadOrder, status: payloadStatus, success: payloadSuccess, url } = payload\n          const limit = payloadLimit ?? 1\n          const order = payloadOrder ?? 'desc'\n          const offset = payloadOffset ?? 0\n          const success = payloadSuccess\n          const status = payloadStatus\n          const fields: Partial<ImageThumbnailResultQuery> = { ...{ limit, offset, order, status, success, url } }\n          const query = new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema }).fields(fields).build()\n          return await diviner.divine([query])\n        }),\n      )\n    )\n      .flat()\n      .filter(isImageThumbnailResult)\n    return results\n  }\n\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ImageThumbnailDivinerState | undefined> {\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState)\n      if (payload) {\n        return payload.state as ImageThumbnailDivinerState\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,cAAc;AACvB,SAAS,uBAAuB;AAEhC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAAS,qBAAqB;AAC9B,SAA0C,sCAAsC;AAChF,SAAS,2BAA2B;AACpC,SAAqC,iCAAiC;AACtE,SAAS,sBAAsB;AAC/B;AAAA,EAGE;AAAA,EAGA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,eAA4B,yBAA0C;AAC/E,SAAS,sBAAsB;AAE/B,SAAS,aAAwB,uBAAuB;AAyBxD,IAAM,aAAa;AAEZ,IAAM,wBAAN,MAAM,+BAAyG,gBAAyB;AAAA,EAC7I,OAAgB,gBAAgB,CAAC,mCAAmC,mBAAmB;AAAA,EAE/E;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA,EAEA,aAAqB,cAAc,iBAAoC,WAA8B;AAWnG,UAAM,iBACJ,MAAM,QAAQ;AAAA,MACZ,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,oBAAoB,aAAa,MAAM;AACnF,cAAM,UAAU,MAAM,UAAU,IAAI,CAAC,oBAAoB,aAAa,CAAC;AACvE,cAAM,wBAAwB,QAAQ,KAAK,gBAAgB;AAC3D,cAAM,mBAAmB,QAAQ,KAAK,WAAW;AACjD,YAAI,CAAC,yBAAyB,CAAC,kBAAkB;AAC/C,kBAAQ,IAAI,2CAA2C;AACvD,iBAAO;AAAA,QACT;AACA,cAAM,+BAA+B,MAAM,cAAc,UAAU,qBAAqB;AACxF,cAAM,0BAA0B,MAAM,cAAc,UAAU,gBAAgB;AAC9E,YAAI,uBAAuB,gCAAgC,kBAAkB;AAAyB,iBAAO;AAC7G,eAAO,CAAC,kBAAkB,oBAAoB,uBAAuB,eAAe,gBAAgB;AAAA,MACtG,CAAC;AAAA,IACH,GACA,OAAO,MAAM;AACf,WAAO;AAAA,EACT;AAAA,EAEA,aAAqB,gBAAgB,OAAkB;AAErD,UAAM,mBACJ,MAAM,QAAQ;AAAA,MACZ,MAAM,OAAO,cAAc,EAAE,IAAI,OAAO,OAAO;AAvGvD;AAwGU,cAAM,yBAAwB,QAAG,oBAAH,mBAC1B,IAAI,CAAC,QAAQ,UAAW,WAAW,uBAAuB,QAAQ,QACnE,OAAO;AACV,cAAM,kBAAiB,QAAG,oBAAH,mBAAoB,UAAU,CAAC,WAAW,WAAW;AAC5E,YAAI,CAAC,sBAAsB,UAAU,mBAAmB;AAAI,iBAAO;AACnE,cAAM,kBAAkB,GAAG,eAAe,IAAI,CAAC,MAAM,UAAW,sBAAsB,SAAS,KAAK,IAAI,OAAO,MAAU,EAAE,OAAO,MAAM;AACxI,cAAM,iBAAgB,QAAG,mBAAH,mBAAoB;AAC1C,cAAM,mBAAmB,MAAM,cAAc,UAAU,EAAE;AACzD,eAAO,gBAAgB,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,oBAAoB,aAAa,CAAU;AAAA,MACnH,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,MAAM;AAChB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAa,MAAM,KAAK,cAAc,KAAM,EAAE,QAAQ,EAAE;AAC9D,UAAM,EAAE,OAAO,IAAI;AAEnB,UAAM,sBAAsB,MAAM,KAAK,+BAA+B,gBAAgB;AACtF,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,OAAO,KAAK;AAAA,MACZ;AAAA,MACA,OAAO;AAAA,MACP,iBAAiB,CAAC,sBAAsB,eAAe;AAAA,IACzD,CAAC,EACA,MAAM;AACT,UAAM,QAAQ,MAAM,oBAAoB,OAAO,CAAC,KAAK,CAAC;AACtD,QAAI,MAAM,WAAW;AAAG;AACxB,UAAM,kBAAkB,MAAM,uBAAsB,gBAAgB,KAAK;AACzE,UAAM,YAAY,MAAM,KAAK,qBAAqB,gBAAgB;AAClE,UAAM,gBAAgB,MAAM,uBAAsB,cAAc,iBAAiB,SAAS;AAE1F,UAAM,UAAkC,cAAc;AAAA,MACpD,CAAC,CAAC,kBAAkB,eAAe,kBAAkB,eAAe,gBAAgB,MAAM;AAlJhG;AAmJQ,cAAM,EAAE,WAAW,IAAI,IAAI;AAC3B,cAAM,EAAE,UAAU,IAAI;AACtB,cAAM,UAAS,sBAAiB,SAAjB,mBAAuB;AAEtC,cAAM,UAAU,CAAC,CAAC,iBAAiB;AACnC,cAAM,UAAU,CAAC,kBAAkB,eAAe,aAAa;AAC/D,cAAM,SAAS,EAAE,SAAS,QAAQ,SAAS,WAAW,IAAI;AAC1D,cAAM,SAAS,IAAI,eAAqC,EAAE,QAAQ,gCAAgC,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AAC1H,eAAO;AAAA,MACT;AAAA,IACF;AAEA,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,UAAM,aAAa,SAAS,MAAM;AAClC,UAAM,eAAe,EAAE,GAAG,WAAW,QAAQ,WAAW;AACxD,UAAM,KAAK,YAAY,YAAY;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAgB,YAAY,OAAmC;AAC7D,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,UAAU,IAAI,eAAwD,EAAE,QAAQ,kBAAkB,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM;AACnI,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC7F,UAAM,UAAU,OAAO,CAAC,IAAI,OAAO,CAAC;AAAA,EACtC;AAAA,EAEA,MAAyB,cAAc,WAAsB,CAAC,GAAoC;AAChG,UAAM,OAAO,SAAS,OAAO,4BAA4B;AACzD,UAAM,UAAU,MAAM,KAAK,0BAA0B,YAAY;AACjE,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,KAAK,IAAI,OAAO,YAAY;AAC1B,cAAM,EAAE,OAAO,cAAc,QAAQ,eAAe,OAAO,cAAc,QAAQ,eAAe,SAAS,gBAAgB,IAAI,IAAI;AACjI,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,QAAQ,gBAAgB;AAC9B,cAAM,SAAS,iBAAiB;AAChC,cAAM,UAAU;AAChB,cAAM,SAAS;AACf,cAAM,SAA6C,EAAE,GAAG,EAAE,OAAO,QAAQ,OAAO,QAAQ,SAAS,IAAI,EAAE;AACvG,cAAM,QAAQ,IAAI,eAA0C,EAAE,QAAQ,0BAA0B,CAAC,EAAE,OAAO,MAAM,EAAE,MAAM;AACxH,eAAO,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AAAA,MACrC,CAAC;AAAA,IACH,GAEC,KAAK,EACL,OAAO,sBAAsB;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,MAAgB,qBAAqB,OAAoB;AA3M3D;AA4MI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA,EAEA,MAAgB,+BAA+B,OAAoB;AAjNrE;AAkNI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA,EAEA,MAAgB,0BAA0B,OAAoB;AAvNhE;AAwNI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAiE;AAC/E,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AAtPzB;AAsP6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,oBAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAa;AAChE,UAAI,SAAS;AACX,eAAO,QAAQ;AAAA,MACjB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA,EAEQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}

package/package.json

@@ -10,32 +10,33 @@
     "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js/issues"
   },
   "dependencies": {
-    "@xylabs/assert": "^2.12.30",
-    "@xylabs/exists": "^2.12.30",
-    "@xyo-network/abstract-diviner": "~2.77.4",
-    "@xyo-network/archivist-wrapper": "~2.77.4",
-    "@xyo-network/boundwitness-model": "~2.77.4",
-    "@xyo-network/core": "~2.77.4",
-    "@xyo-network/diviner-boundwitness-model": "~2.77.4",
-    "@xyo-network/diviner-model": "~2.77.4",
-    "@xyo-network/diviner-payload-model": "~2.77.4",
-    "@xyo-network/diviner-wrapper": "~2.77.4",
-    "@xyo-network/image-thumbnail-payload-plugin": "~2.77.4",
-    "@xyo-network/module-model": "~2.77.4",
-    "@xyo-network/payload-builder": "~2.77.4",
-    "@xyo-network/payload-model": "~2.77.4",
-    "@xyo-network/witness-timestamp": "~2.77.4"
+    "@xylabs/assert": "^2.13.3",
+    "@xylabs/exists": "^2.13.3",
+    "@xyo-network/abstract-diviner": "~2.77.5",
+    "@xyo-network/archivist-model": "~2.77.5",
+    "@xyo-network/archivist-wrapper": "~2.77.5",
+    "@xyo-network/boundwitness-model": "~2.77.5",
+    "@xyo-network/core": "~2.77.5",
+    "@xyo-network/diviner-boundwitness-model": "~2.77.5",
+    "@xyo-network/diviner-model": "~2.77.5",
+    "@xyo-network/diviner-payload-model": "~2.77.5",
+    "@xyo-network/diviner-wrapper": "~2.77.5",
+    "@xyo-network/image-thumbnail-payload-plugin": "~2.77.5",
+    "@xyo-network/module-model": "~2.77.5",
+    "@xyo-network/payload-builder": "~2.77.5",
+    "@xyo-network/payload-model": "~2.77.5",
+    "@xyo-network/witness-timestamp": "~2.77.5"
   },
   "devDependencies": {
-    "@xylabs/delay": "^2.12.30",
-    "@xylabs/ts-scripts-yarn3": "^3.1.12",
-    "@xylabs/tsconfig": "^3.1.12",
-    "@xyo-network/account": "~2.77.4",
-    "@xyo-network/boundwitness-builder": "~2.77.4",
-    "@xyo-network/diviner-boundwitness-memory": "~2.77.4",
-    "@xyo-network/diviner-payload-memory": "~2.77.4",
-    "@xyo-network/memory-archivist": "~2.77.4",
-    "@xyo-network/node-memory": "~2.77.4",
+    "@xylabs/delay": "^2.13.3",
+    "@xylabs/ts-scripts-yarn3": "^3.1.13",
+    "@xylabs/tsconfig": "^3.1.13",
+    "@xyo-network/account": "~2.77.5",
+    "@xyo-network/boundwitness-builder": "~2.77.5",
+    "@xyo-network/diviner-boundwitness-memory": "~2.77.5",
+    "@xyo-network/diviner-payload-memory": "~2.77.5",
+    "@xyo-network/memory-archivist": "~2.77.5",
+    "@xyo-network/node-memory": "~2.77.5",
     "typescript": "^5.2.2"
   },
   "description": "Primary SDK for using XYO Protocol 2.0",
@@ -78,5 +79,5 @@
     "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
   },
   "sideEffects": false,
-  "version": "2.77.4"
+  "version": "2.77.5"
 }

package/src/Diviner/Diviner.ts

@@ -1,6 +1,7 @@
 import { assertEx } from '@xylabs/assert'
 import { exists } from '@xylabs/exists'
 import { AbstractDiviner } from '@xyo-network/abstract-diviner'
+import { ArchivistInstance } from '@xyo-network/archivist-model'
 import { ArchivistWrapper } from '@xyo-network/archivist-wrapper'
 import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'
 import { isBoundWitness } from '@xyo-network/boundwitness-model'
@@ -48,6 +49,8 @@ type ImageThumbnailResultQuery = PayloadDivinerQueryPayload & { schemas: [ImageT
     QueryableImageThumbnailResultProperties
   >
 
+type IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>
+
 const moduleName = 'ImageThumbnailDiviner'
 
 export class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {
@@ -63,46 +66,7 @@ export class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams =
     return this.config.pollFrequency ?? 10_000
   }
 
-  /**
-   * Works in the background to populate index for the Diviner
-   * @returns
-   */
-  protected backgroundDivine = async (): Promise<void> => {
-    // Load last state
-    const lastState = (await this.retrieveState()) ?? { offset: 0 }
-    const { offset } = lastState
-    // Get next batch of results
-    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')
-    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
-      .fields({
-        limit: this.payloadDivinerLimit,
-        offset,
-        order: 'asc',
-        payload_schemas: [ImageThumbnailSchema, TimestampSchema],
-      })
-      .build()
-    const batch = await boundWitnessDiviner.divine([query])
-    if (batch.length === 0) return
-    // Find all the indexable hashes in this batch
-    type IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>
-    const indexableHashes: IndexableHashes[] = (
-      await Promise.all(
-        batch.filter(isBoundWitness).map(async (bw) => {
-          const imageThumbnailIndexes = bw.payload_schemas
-            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))
-            .filter(exists)
-          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)
-          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined
-          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)
-          const timestampHash = bw.payload_hashes?.[timestampIndex]
-          const boundWitnessHash = await PayloadHasher.hashAsync(bw)
-          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)
-        }),
-      )
-    )
-      .flat()
-      .filter(exists)
-    const archivist = await this.getArchivistForStore('thumbnailStore')
+  private static async indexableData(indexableHashes: IndexableHashes[], archivist: ArchivistInstance) {
     // Find all the indexable data associated with the indexable hashes
     type IndexableData = Readonly<
       [
@@ -130,6 +94,54 @@ export class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams =
         }),
       )
     ).filter(exists)
+    return indexableData
+  }
+
+  private static async indexableHashes(batch: Payload[]) {
+    // Find all the indexable hashes in this batch
+    const indexableHashes: IndexableHashes[] = (
+      await Promise.all(
+        batch.filter(isBoundWitness).map(async (bw) => {
+          const imageThumbnailIndexes = bw.payload_schemas
+            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))
+            .filter(exists)
+          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)
+          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined
+          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)
+          const timestampHash = bw.payload_hashes?.[timestampIndex]
+          const boundWitnessHash = await PayloadHasher.hashAsync(bw)
+          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)
+        }),
+      )
+    )
+      .flat()
+      .filter(exists)
+    return indexableHashes
+  }
+
+  /**
+   * Works in the background to populate index for the Diviner
+   * @returns
+   */
+  protected backgroundDivine = async (): Promise<void> => {
+    // Load last state
+    const lastState = (await this.retrieveState()) ?? { offset: 0 }
+    const { offset } = lastState
+    // Get next batch of results
+    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')
+    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
+      .fields({
+        limit: this.payloadDivinerLimit,
+        offset,
+        order: 'asc',
+        payload_schemas: [ImageThumbnailSchema, TimestampSchema],
+      })
+      .build()
+    const batch = await boundWitnessDiviner.divine([query])
+    if (batch.length === 0) return
+    const indexableHashes = await ImageThumbnailDiviner.indexableHashes(batch)
+    const archivist = await this.getArchivistForStore('thumbnailStore')
+    const indexableData = await ImageThumbnailDiviner.indexableData(indexableHashes, archivist)
     // Build index results from the indexable data
     const indexes: ImageThumbnailResult[] = indexableData.map(
       ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {