@xyo-network/archivist-mongodb 3.1.5 → 3.1.7

package/dist/browser/Archivist.d.ts

@@ -1,5 +1,6 @@
 import type { Hash } from '@xylabs/hex';
 import { AbstractArchivist } from '@xyo-network/archivist-abstract';
+import type { ArchivistNextOptions } from '@xyo-network/archivist-model';
 import type { Payload, Schema, WithMeta } from '@xyo-network/payload-model';
 import type { PayloadWithMongoMeta } from '@xyo-network/payload-mongodb';
 declare const MongoDBArchivistBase: (abstract new (...args: any[]) => {
@@ -67,9 +68,15 @@ export declare class MongoDBArchivist extends MongoDBArchivistBase {
     static readonly configSchemas: Schema[];
     static readonly defaultConfigSchema: Schema;
     readonly queries: string[];
+    /**
+     * The amount of time to allow the aggregate query to execute
+     */
+    protected readonly aggregateTimeoutMs = 10000;
     head(): Promise<Payload | undefined>;
+    protected findOneByHash(hash: Hash): Promise<import("mongodb").WithId<PayloadWithMongoMeta> | undefined>;
     protected getHandler(hashes: Hash[]): Promise<WithMeta<Payload>[]>;
     protected insertHandler(payloads: Payload[]): Promise<WithMeta<Payload>[]>;
+    protected nextHandler(options?: ArchivistNextOptions): Promise<WithMeta<Payload>[]>;
     protected startHandler(): Promise<boolean>;
 }
 export {};

package/dist/browser/Archivist.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Archivist.d.ts","sourceRoot":"","sources":["../../src/Archivist.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AAKnE,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAAE,QAAQ,EAC1B,MAAM,4BAA4B,CAAA;AACnC,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAA;AAMxE,QAAA,MAAM,oBAAoB;;;;;8BAgCoD,8BAA8B;;;;oBAOtE,aACnC;;+CA/B0E,+BACnE;0CAGQ,+BAA+B;uBACtB,mCAEJ;;;;;;;qBAwB2D,2BAA2B;;yBA7CrF,4BAA4B;;mCACG,4BAEnC;;uBAE6B,4BAExC,wBAAwB,4BACjC,8BAA8B,2BAA2B,iCAAiC,2BACjF,8BAA8B,4BAA4B,wBAE/D,4BAA4B,8BAA8B,2BAE7D,iCAAiC,2BAC7B;iCAAsD,CAAC,iBAAiB,+BACnE;4BAGR,CAAD,iBAAiB,+BAA+B;2BACtB,mCAEJ;;;;;;;8CAE+B,2BAA2B;;wCAEV,+BAA+B,6CAClF,+BAA+B;0BACV,8BAA8B;mCAErC,+BAA+B,6CAA6C,+BACzF;;+BAiBP,iBAAiB;;6BACA,iCAAiC,6BAA6B,iCAC3E,0CAA0C,2BACxD,wBACkB,2BAA2B,kCAAkC,CAAC,SAAS,4BAC7F,wBAAwB,CAAC,mBAAmB,iBACpC,oBAAoB,2BAA2B;iCACjD,iCAAiC,6BAA6B,iCAAiC,4CAClF,2BAA2B,wBAAwB,2BAGrE,kCAAkC,CAAC,SAAS,4BAA4B,wBAAwB,CAAC,qBAGxF,iBAAiB;yBAEvB,iBAAa;wBAIN,iBAAiB;sBAA6C,2BAA2B;4CAAuE,2BAA2B,0CAA0C,2BAA2B;yCAA8E,2BAA2B,oEAAoE,2BAA2B;+CAAsG,2BAA2B,oEAAoE,2BAA2B;2CAAkG,2BAA2B,0CAA0C,2BAA2B;wCAA+E,2BAA2B,mFAAmF,4BAA4B,uBAAuB,2BAA2B;4BAA2E,4BAA4B;uCAAkF,2BAA2B,mFAAmF,4BAA4B,uBAAuB,2BAA2B,gDAAgD,4BAA4B;2BAAuD,4BAA4B,4BAA4B,4BAA4B;yCAAqE,2BAA2B,mEAAmE,4BAA4B,uBAAuB,2BAA2B,gDAAgD,4BAA4B;;;4BA9Dv8D,CAAA;AAElE,qBAAa,gBAAiB,SAAQ,oBAAoB;IACxD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAyD;IACzG,gBAAyB,mBAAmB,EAAE,MAAM,CAA+B;IAEnF,SAAkB,OAAO,EAAE,MAAM,EAAE,CAAiD;IAErE,IAAI,IAAI,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC;cAK1B,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAyBxD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAkBhE,YAAY;CAKtC"}
+{"version":3,"file":"Archivist.d.ts","sourceRoot":"","sources":["../../src/Archivist.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAA;AAKxE,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAAE,QAAQ,EAC1B,MAAM,4BAA4B,CAAA;AACnC,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAA;AAOxE,QAAA,MAAM,oBAAoB;;;;;8BA6CsC,8BAA8B;;;;oBAMzF,aAAa;;+CA7CqE,+BAEnF;0CAGQ,+BAA8B;uBAGtC,mCAA+B;;;;;;;qBAuCnB,2BAA2B;;yBA7DT,4BACR;;mCACkB,4BAA4B;;uBACb,4BAClC,wBAEZ,4BAA4B,8BAEhB,2BAA2B,iCAEnC,2BAA2B,8BAC/B,4BAA4B,wBAAwB,4BAA4B,8BAC9E,2BAA2B,iCAAiC,2BAE5D;iCAAsD,CAAC,iBAAiB,+BAEnF;4BAED,CAAH,iBACY,+BAA8B;2BAGtC,mCAA+B;;;;;;;8CAMd,2BAErB;;wCAC8D,+BAC5C,6CAGV,+BAA6B;0BAE3B,8BACE;mCAC0B,+BAA+B,6CAE7D,+BAKC;;+BAeoG,iBACxG;;6BACqB,iCAAiC,6BAE/C,iCAAiC,0CAA0C,2BAA2B,wBAAwB,2BAC1H,kCAAkC,CAAC,SAAS,4BAC3D,wBAAwB,CAAC,mBAAmB,iBAAiB,oBAAoB,2BAC7E;iCAGgC,iCAAiC,6BACjE,iCAAiC,4CACH,2BAA0B,wBAAwB,2BAA2B,kCAChF,CAAC,SAAS,4BACnB,wBAAwB,CAAC,qBAAqB,iBACrE;yBAAgD,iBAAiB;wBACzD,iBAAa;sBAA6C,2BAEjD;4CAAuE,2BAA2B,0CAC/E,2BAC1B;yCAA8E,2BAC7E,oEAAoE,2BAA2B;+CAItG,2BAAsB,oEAAoE,2BAA2B;2CAGrC,2BAC5E,0CAEF,2BAAqB;wCAIvB,2BAAyB,mFAKxB,4BACU,uBAAuB,2BAC5B;4BACJ,4BAA4B;uCAGM,2BAC7B,mFAC+C,4BACvD,uBAAuB,2BACzB,gDAAgD,4BAE1C;2BAGF,4BAA4B,4BACb,4BAA2B;yCACT,2BAClB,mEAEb,4BACJ,uBAAuB,2BAA2B,gDAE7C,4BAAmB;;;4BApHuC,CAAA;AAElE,qBAAa,gBAAiB,SAAQ,oBAAoB;IACxD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAyD;IACzG,gBAAyB,mBAAmB,EAAE,MAAM,CAA+B;IAEnF,SAAkB,OAAO,EAAE,MAAM,EAAE,CAA2E;IAE9G;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,kBAAkB,SAAS;IAE/B,IAAI,IAAI,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC;cAKnC,aAAa,CAAC,IAAI,EAAE,IAAI;cAoBf,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAyBxD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAkBhE,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAkEzE,YAAY;CAKtC"}

package/dist/browser/index.mjs

@@ -4,12 +4,13 @@ var __name = (target, value) => __defProp(target, "name", { value, configurable:
 // src/Archivist.ts
 import { exists } from "@xylabs/exists";
 import { AbstractArchivist } from "@xyo-network/archivist-abstract";
-import { ArchivistInsertQuerySchema } from "@xyo-network/archivist-model";
+import { ArchivistInsertQuerySchema, ArchivistNextQuerySchema } from "@xyo-network/archivist-model";
 import { MongoDBArchivistConfigSchema } from "@xyo-network/archivist-model-mongodb";
 import { MongoDBModuleMixin } from "@xyo-network/module-abstract-mongodb";
 import { PayloadBuilder } from "@xyo-network/payload-builder";
 import { fromDbRepresentation, toDbRepresentation } from "@xyo-network/payload-mongodb";
 import { PayloadWrapper as PayloadWrapper2 } from "@xyo-network/payload-wrapper";
+import { ObjectId } from "mongodb";
 
 // src/lib/validByType.ts
 import { isBoundWitness, isQueryBoundWitness } from "@xyo-network/boundwitness-model";
@@ -25,7 +26,7 @@ var validByType = /* @__PURE__ */ __name(async (payloads = []) => {
       const wrapper = isQueryBoundWitness(payload) ? QueryBoundWitnessWrapper : BoundWitnessWrapper;
       const bw = wrapper.parse(payload);
       if (await bw.getValid()) {
-        results[0].push(bw.payload);
+        results[0].push(payload);
       } else {
         const errors = await bw.getErrors();
         console.log(`validByType.Error: ${JSON.stringify(errors, null, 2)}`);
@@ -33,7 +34,7 @@ var validByType = /* @__PURE__ */ __name(async (payloads = []) => {
     } else {
       const payloadWrapper = PayloadWrapper.wrap(payload);
       if (await payloadWrapper.getValid()) {
-        results[1].push(payloadWrapper.payload);
+        results[1].push(payload);
       }
     }
     return;
@@ -54,14 +55,47 @@ var MongoDBArchivist = class extends MongoDBArchivistBase {
   static defaultConfigSchema = MongoDBArchivistConfigSchema;
   queries = [
     ArchivistInsertQuerySchema,
+    ArchivistNextQuerySchema,
     ...super.queries
   ];
+  /**
+  * The amount of time to allow the aggregate query to execute
+  */
+  aggregateTimeoutMs = 1e4;
   async head() {
-    const head = await (await this.payloads.find({})).sort({
-      _timestamp: -1
-    }).limit(1).toArray();
+    const head = await this.next({
+      limit: 1,
+      order: "desc"
+    });
     return head[0] ? PayloadWrapper2.wrap(head[0]).payload : void 0;
   }
+  async findOneByHash(hash) {
+    const dataPayload = await this.payloads.findOne({
+      _$hash: hash
+    });
+    if (dataPayload) {
+      return dataPayload;
+    } else {
+      const dataBw = await this.boundWitnesses.findOne({
+        _$hash: hash
+      });
+      if (dataBw) {
+        return dataBw;
+      } else {
+        const payload = await this.payloads.findOne({
+          _hash: hash
+        });
+        if (payload) {
+          return payload;
+        } else {
+          const bw = await this.boundWitnesses.findOne({
+            _hash: hash
+          });
+          return bw ?? void 0;
+        }
+      }
+    }
+  }
   async getHandler(hashes) {
     let remainingHashes = [
       ...hashes
@@ -100,17 +134,92 @@ var MongoDBArchivist = class extends MongoDBArchivistBase {
     const payloadsWithExternalMeta = await Promise.all(p.map((value, index) => toDbRepresentation(value, index)));
     if (payloadsWithExternalMeta.length > 0) {
       const payloadsResult = await this.payloads.insertMany(payloadsWithExternalMeta);
-      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length) throw new Error("MongoDBDeterministicArchivist: Error inserting Payloads");
+      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length) throw new Error("MongoDBArchivist: Error inserting Payloads");
     }
     const boundWitnessesWithExternalMeta = await Promise.all(bw.map((value, index) => toDbRepresentation(value, index)));
     if (boundWitnessesWithExternalMeta.length > 0) {
       const boundWitnessesResult = await this.boundWitnesses.insertMany(boundWitnessesWithExternalMeta);
-      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length) throw new Error("MongoDBDeterministicArchivist: Error inserting BoundWitnesses");
+      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length) throw new Error("MongoDBArchivist: Error inserting BoundWitnesses");
     }
     return await PayloadBuilder.build([
       ...boundWitnessesWithExternalMeta,
       ...payloadsWithExternalMeta
-    ]);
+    ].map(fromDbRepresentation));
+  }
+  async nextHandler(options) {
+    let { limit, offset, order } = options ?? {
+      limit: 10,
+      order: "desc"
+    };
+    if (!limit) limit = 10;
+    if (limit > 100) limit = 100;
+    if (order != "asc") order = "desc";
+    let id;
+    if (offset) {
+      const payload = await this.findOneByHash(offset);
+      if (payload) id = payload._id;
+    } else {
+      id = order === "asc" ? ObjectId.createFromTime(0) : ObjectId.createFromTime((Date.now() + 1e4) / 1e3);
+    }
+    if (!id) return [];
+    const sort = order === "asc" ? 1 : -1;
+    const match = order === "asc" ? {
+      _id: {
+        $gt: id
+      }
+    } : {
+      _id: {
+        $lt: id
+      }
+    };
+    const foundPayloads = await this.payloads.useCollection((collection) => {
+      return collection.aggregate([
+        // Pre-filter payloads collection
+        {
+          $match: match
+        },
+        // Sort payloads by _id
+        {
+          $sort: {
+            _id: sort
+          }
+        },
+        // Limit payloads to the first N payloads
+        {
+          $limit: limit
+        },
+        // Combine with filtered boundWitnesses collection
+        {
+          $unionWith: {
+            coll: this.boundWitnessSdkConfig.collection,
+            pipeline: [
+              {
+                $match: match
+              },
+              {
+                $sort: {
+                  _id: sort
+                }
+              },
+              {
+                $limit: limit
+              }
+            ]
+          }
+        },
+        // Sort the combined result by _id
+        {
+          $sort: {
+            _id: sort
+          }
+        },
+        // Limit the final result to N documents
+        {
+          $limit: limit
+        }
+      ]).maxTimeMS(this.aggregateTimeoutMs).toArray();
+    });
+    return await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation));
   }
   async startHandler() {
     await super.startHandler();

package/dist/browser/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Archivist.ts","../../src/lib/validByType.ts","../../src/index.ts"],"sourcesContent":["import { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport { ArchivistInsertQuerySchema } from '@xyo-network/archivist-model'\nimport { MongoDBArchivistConfigSchema } from '@xyo-network/archivist-model-mongodb'\nimport { MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema, WithMeta,\n} from '@xyo-network/payload-model'\nimport type { PayloadWithMongoMeta } from '@xyo-network/payload-mongodb'\nimport { fromDbRepresentation, toDbRepresentation } from '@xyo-network/payload-mongodb'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\n\nimport { validByType } from './lib/index.js'\n\nconst MongoDBArchivistBase = MongoDBModuleMixin(AbstractArchivist)\n\nexport class MongoDBArchivist extends MongoDBArchivistBase {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, MongoDBArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = MongoDBArchivistConfigSchema\n\n  override readonly queries: string[] = [ArchivistInsertQuerySchema, ...super.queries]\n\n  override async head(): Promise<Payload | undefined> {\n    const head = await (await this.payloads.find({})).sort({ _timestamp: -1 }).limit(1).toArray()\n    return head[0] ? PayloadWrapper.wrap(head[0]).payload : undefined\n  }\n\n  protected override async getHandler(hashes: Hash[]): Promise<WithMeta<Payload>[]> {\n    let remainingHashes = [...hashes]\n\n    const dataPayloads = (await Promise.all(remainingHashes.map(_$hash => this.payloads.findOne({ _$hash })))).filter(exists)\n    const dataPayloadsHashes = new Set(dataPayloads.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataPayloadsHashes.has(hash))\n\n    const dataBws = (await Promise.all(remainingHashes.map(_$hash => this.boundWitnesses.findOne({ _$hash })))).filter(exists)\n    const dataBwsHashes = new Set(dataBws.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataBwsHashes.has(hash))\n\n    const payloads = (await Promise.all(remainingHashes.map(_hash => this.payloads.findOne({ _hash })))).filter(exists)\n    const payloadsHashes = new Set(payloads.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !payloadsHashes.has(hash))\n\n    const bws = (await Promise.all(remainingHashes.map(_hash => this.boundWitnesses.findOne({ _hash })))).filter(exists)\n    const bwsHashes = new Set(bws.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !bwsHashes.has(hash))\n\n    const foundPayloads = [...dataPayloads, ...dataBws, ...payloads, ...bws] as PayloadWithMongoMeta<Payload & { _$hash: Hash; _$meta?: unknown }>[]\n    const result = await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))\n    // console.log(`getHandler: ${JSON.stringify(hashes, null, 2)}:${JSON.stringify(result, null, 2)}`)\n    return result\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<WithMeta<Payload>[]> {\n    const [bw, p] = await validByType(payloads)\n    const payloadsWithExternalMeta = await Promise.all(p.map((value, index) => toDbRepresentation(value, index)))\n    if (payloadsWithExternalMeta.length > 0) {\n      const payloadsResult = await this.payloads.insertMany(payloadsWithExternalMeta)\n      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length)\n        throw new Error('MongoDBDeterministicArchivist: Error inserting Payloads')\n    }\n    const boundWitnessesWithExternalMeta = await Promise.all(bw.map((value, index) => toDbRepresentation(value, index)))\n    if (boundWitnessesWithExternalMeta.length > 0) {\n      const boundWitnessesResult = await this.boundWitnesses.insertMany(boundWitnessesWithExternalMeta)\n      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length)\n        throw new Error('MongoDBDeterministicArchivist: Error inserting BoundWitnesses')\n    }\n\n    return await PayloadBuilder.build([...boundWitnessesWithExternalMeta, ...payloadsWithExternalMeta])\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    await this.ensureIndexes()\n    return true\n  }\n}\n","import type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness, isQueryBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessWrapper, QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-wrapper'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\n\nexport const validByType = async (payloads: Payload[] = []) => {\n  const results: [BoundWitness[], Payload[]] = [[], []]\n  await Promise.all(\n    payloads.map(async (payload) => {\n      if (isBoundWitness(payload)) {\n        const wrapper = isQueryBoundWitness(payload) ? QueryBoundWitnessWrapper : BoundWitnessWrapper\n        const bw = wrapper.parse(payload)\n        if (await bw.getValid()) {\n          results[0].push(bw.payload)\n        } else {\n          const errors = await bw.getErrors()\n          console.log(`validByType.Error: ${JSON.stringify(errors, null, 2)}`)\n        }\n      } else {\n        const payloadWrapper = PayloadWrapper.wrap(payload)\n        if (await payloadWrapper.getValid()) {\n          results[1].push(payloadWrapper.payload)\n        }\n      }\n      return\n    }),\n  )\n  return results\n}\n","export * from './Archivist.js'\nexport * from '@xyo-network/archivist-model-mongodb'\n"],"mappings":";;;;AAAA,SAASA,cAAc;AAEvB,SAASC,yBAAyB;AAClC,SAASC,kCAAkC;AAC3C,SAASC,oCAAoC;AAC7C,SAASC,0BAA0B;AACnC,SAASC,sBAAsB;AAK/B,SAASC,sBAAsBC,0BAA0B;AACzD,SAASC,kBAAAA,uBAAsB;;;ACX/B,SAASC,gBAAgBC,2BAA2B;AACpD,SAASC,qBAAqBC,gCAAgC;AAE9D,SAASC,sBAAsB;AAExB,IAAMC,cAAc,8BAAOC,WAAsB,CAAA,MAAE;AACxD,QAAMC,UAAuC;IAAC,CAAA;IAAI,CAAA;;AAClD,QAAMC,QAAQC,IACZH,SAASI,IAAI,OAAOC,YAAAA;AAClB,QAAIC,eAAeD,OAAAA,GAAU;AAC3B,YAAME,UAAUC,oBAAoBH,OAAAA,IAAWI,2BAA2BC;AAC1E,YAAMC,KAAKJ,QAAQK,MAAMP,OAAAA;AACzB,UAAI,MAAMM,GAAGE,SAAQ,GAAI;AACvBZ,gBAAQ,CAAA,EAAGa,KAAKH,GAAGN,OAAO;MAC5B,OAAO;AACL,cAAMU,SAAS,MAAMJ,GAAGK,UAAS;AACjCC,gBAAQC,IAAI,sBAAsBC,KAAKC,UAAUL,QAAQ,MAAM,CAAA,CAAA,EAAI;MACrE;IACF,OAAO;AACL,YAAMM,iBAAiBC,eAAeC,KAAKlB,OAAAA;AAC3C,UAAI,MAAMgB,eAAeR,SAAQ,GAAI;AACnCZ,gBAAQ,CAAA,EAAGa,KAAKO,eAAehB,OAAO;MACxC;IACF;AACA;EACF,CAAA,CAAA;AAEF,SAAOJ;AACT,GAvB2B;;;ADU3B,IAAMuB,uBAAuBC,mBAAmBC,iBAAAA;AAEzC,IAAMC,mBAAN,cAA+BH,qBAAAA;EAlBtC,OAkBsCA;;;EACpC,OAAyBI,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAErCE,UAAoB;IAACC;OAA+B,MAAMD;;EAE5E,MAAeE,OAAqC;AAClD,UAAMA,OAAO,OAAO,MAAM,KAAKC,SAASC,KAAK,CAAC,CAAA,GAAIC,KAAK;MAAEC,YAAY;IAAG,CAAA,EAAGC,MAAM,CAAA,EAAGC,QAAO;AAC3F,WAAON,KAAK,CAAA,IAAKO,gBAAeC,KAAKR,KAAK,CAAA,CAAE,EAAES,UAAUC;EAC1D;EAEA,MAAyBC,WAAWC,QAA8C;AAChF,QAAIC,kBAAkB;SAAID;;AAE1B,UAAME,gBAAgB,MAAMC,QAAQC,IAAIH,gBAAgBI,IAAIC,CAAAA,WAAU,KAAKjB,SAASkB,QAAQ;MAAED;IAAO,CAAA,CAAA,CAAA,GAAME,OAAOC,MAAAA;AAClH,UAAMC,qBAAqB,IAAIC,IAAIT,aAAaG,IAAIR,CAAAA,YAAWA,QAAQS,MAAM,CAAA;AAC7EL,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACF,mBAAmBG,IAAID,IAAAA,CAAAA;AAEzE,UAAME,WAAW,MAAMX,QAAQC,IAAIH,gBAAgBI,IAAIC,CAAAA,WAAU,KAAKS,eAAeR,QAAQ;MAAED;IAAO,CAAA,CAAA,CAAA,GAAME,OAAOC,MAAAA;AACnH,UAAMO,gBAAgB,IAAIL,IAAIG,QAAQT,IAAIR,CAAAA,YAAWA,QAAQS,MAAM,CAAA;AACnEL,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACI,cAAcH,IAAID,IAAAA,CAAAA;AAEpE,UAAMvB,YAAY,MAAMc,QAAQC,IAAIH,gBAAgBI,IAAIY,CAAAA,UAAS,KAAK5B,SAASkB,QAAQ;MAAEU;IAAM,CAAA,CAAA,CAAA,GAAMT,OAAOC,MAAAA;AAC5G,UAAMS,iBAAiB,IAAIP,IAAItB,SAASgB,IAAIR,CAAAA,YAAWA,QAAQoB,KAAK,CAAA;AACpEhB,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACM,eAAeL,IAAID,IAAAA,CAAAA;AAErE,UAAMO,OAAO,MAAMhB,QAAQC,IAAIH,gBAAgBI,IAAIY,CAAAA,UAAS,KAAKF,eAAeR,QAAQ;MAAEU;IAAM,CAAA,CAAA,CAAA,GAAMT,OAAOC,MAAAA;AAC7G,UAAMW,YAAY,IAAIT,IAAIQ,IAAId,IAAIR,CAAAA,YAAWA,QAAQoB,KAAK,CAAA;AAC1DhB,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACQ,UAAUP,IAAID,IAAAA,CAAAA;AAEhE,UAAMS,gBAAgB;SAAInB;SAAiBY;SAAYzB;SAAa8B;;AACpE,UAAMG,SAAS,MAAMC,eAAeC,MAAMH,cAAchB,IAAIoB,oBAAAA,CAAAA;AAE5D,WAAOH;EACT;EAEA,MAAyBI,cAAcrC,UAAmD;AACxF,UAAM,CAACsC,IAAIC,CAAAA,IAAK,MAAMC,YAAYxC,QAAAA;AAClC,UAAMyC,2BAA2B,MAAM3B,QAAQC,IAAIwB,EAAEvB,IAAI,CAAC0B,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AACrG,QAAIF,yBAAyBI,SAAS,GAAG;AACvC,YAAMC,iBAAiB,MAAM,KAAK9C,SAAS+C,WAAWN,wBAAAA;AACtD,UAAI,CAACK,eAAeE,gBAAgBF,eAAeG,kBAAkBR,yBAAyBI,OAC5F,OAAM,IAAIK,MAAM,yDAAA;IACpB;AACA,UAAMC,iCAAiC,MAAMrC,QAAQC,IAAIuB,GAAGtB,IAAI,CAAC0B,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AAC5G,QAAIQ,+BAA+BN,SAAS,GAAG;AAC7C,YAAMO,uBAAuB,MAAM,KAAK1B,eAAeqB,WAAWI,8BAAAA;AAClE,UAAI,CAACC,qBAAqBJ,gBAAgBI,qBAAqBH,kBAAkBE,+BAA+BN,OAC9G,OAAM,IAAIK,MAAM,+DAAA;IACpB;AAEA,WAAO,MAAMhB,eAAeC,MAAM;SAAIgB;SAAmCV;KAAyB;EACpG;EAEA,MAAyBY,eAAe;AACtC,UAAM,MAAMA,aAAAA;AACZ,UAAM,KAAKC,cAAa;AACxB,WAAO;EACT;AACF;;;AE5EA,cAAc;","names":["exists","AbstractArchivist","ArchivistInsertQuerySchema","MongoDBArchivistConfigSchema","MongoDBModuleMixin","PayloadBuilder","fromDbRepresentation","toDbRepresentation","PayloadWrapper","isBoundWitness","isQueryBoundWitness","BoundWitnessWrapper","QueryBoundWitnessWrapper","PayloadWrapper","validByType","payloads","results","Promise","all","map","payload","isBoundWitness","wrapper","isQueryBoundWitness","QueryBoundWitnessWrapper","BoundWitnessWrapper","bw","parse","getValid","push","errors","getErrors","console","log","JSON","stringify","payloadWrapper","PayloadWrapper","wrap","MongoDBArchivistBase","MongoDBModuleMixin","AbstractArchivist","MongoDBArchivist","configSchemas","MongoDBArchivistConfigSchema","defaultConfigSchema","queries","ArchivistInsertQuerySchema","head","payloads","find","sort","_timestamp","limit","toArray","PayloadWrapper","wrap","payload","undefined","getHandler","hashes","remainingHashes","dataPayloads","Promise","all","map","_$hash","findOne","filter","exists","dataPayloadsHashes","Set","hash","has","dataBws","boundWitnesses","dataBwsHashes","_hash","payloadsHashes","bws","bwsHashes","foundPayloads","result","PayloadBuilder","build","fromDbRepresentation","insertHandler","bw","p","validByType","payloadsWithExternalMeta","value","index","toDbRepresentation","length","payloadsResult","insertMany","acknowledged","insertedCount","Error","boundWitnessesWithExternalMeta","boundWitnessesResult","startHandler","ensureIndexes"]}
+{"version":3,"sources":["../../src/Archivist.ts","../../src/lib/validByType.ts","../../src/index.ts"],"sourcesContent":["import { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport type { ArchivistNextOptions } from '@xyo-network/archivist-model'\nimport { ArchivistInsertQuerySchema, ArchivistNextQuerySchema } from '@xyo-network/archivist-model'\nimport { MongoDBArchivistConfigSchema } from '@xyo-network/archivist-model-mongodb'\nimport { MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema, WithMeta,\n} from '@xyo-network/payload-model'\nimport type { PayloadWithMongoMeta } from '@xyo-network/payload-mongodb'\nimport { fromDbRepresentation, toDbRepresentation } from '@xyo-network/payload-mongodb'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\nimport { ObjectId } from 'mongodb'\n\nimport { validByType } from './lib/index.js'\n\nconst MongoDBArchivistBase = MongoDBModuleMixin(AbstractArchivist)\n\nexport class MongoDBArchivist extends MongoDBArchivistBase {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, MongoDBArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = MongoDBArchivistConfigSchema\n\n  override readonly queries: string[] = [ArchivistInsertQuerySchema, ArchivistNextQuerySchema, ...super.queries]\n\n  /**\n   * The amount of time to allow the aggregate query to execute\n   */\n  protected readonly aggregateTimeoutMs = 10_000\n\n  override async head(): Promise<Payload | undefined> {\n    const head = await this.next({ limit: 1, order: 'desc' })\n    return head[0] ? PayloadWrapper.wrap(head[0]).payload : undefined\n  }\n\n  protected async findOneByHash(hash: Hash) {\n    const dataPayload = (await this.payloads.findOne({ _$hash: hash }))\n    if (dataPayload) {\n      return dataPayload\n    } else {\n      const dataBw = (await this.boundWitnesses.findOne({ _$hash: hash }))\n      if (dataBw) {\n        return dataBw\n      } else {\n        const payload = (await this.payloads.findOne({ _hash: hash }))\n        if (payload) {\n          return payload\n        } else {\n          const bw = (await this.boundWitnesses.findOne({ _hash: hash }))\n          return bw ?? undefined\n        }\n      }\n    }\n  }\n\n  protected override async getHandler(hashes: Hash[]): Promise<WithMeta<Payload>[]> {\n    let remainingHashes = [...hashes]\n\n    const dataPayloads = (await Promise.all(remainingHashes.map(_$hash => this.payloads.findOne({ _$hash })))).filter(exists)\n    const dataPayloadsHashes = new Set(dataPayloads.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataPayloadsHashes.has(hash))\n\n    const dataBws = (await Promise.all(remainingHashes.map(_$hash => this.boundWitnesses.findOne({ _$hash })))).filter(exists)\n    const dataBwsHashes = new Set(dataBws.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataBwsHashes.has(hash))\n\n    const payloads = (await Promise.all(remainingHashes.map(_hash => this.payloads.findOne({ _hash })))).filter(exists)\n    const payloadsHashes = new Set(payloads.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !payloadsHashes.has(hash))\n\n    const bws = (await Promise.all(remainingHashes.map(_hash => this.boundWitnesses.findOne({ _hash })))).filter(exists)\n    const bwsHashes = new Set(bws.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !bwsHashes.has(hash))\n\n    const foundPayloads = [...dataPayloads, ...dataBws, ...payloads, ...bws] as PayloadWithMongoMeta<Payload & { _$hash: Hash; _$meta?: unknown }>[]\n    const result = await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))\n    // console.log(`getHandler: ${JSON.stringify(hashes, null, 2)}:${JSON.stringify(result, null, 2)}`)\n    return result\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<WithMeta<Payload>[]> {\n    const [bw, p] = await validByType(payloads)\n    const payloadsWithExternalMeta = await Promise.all(p.map((value, index) => toDbRepresentation(value, index)))\n    if (payloadsWithExternalMeta.length > 0) {\n      const payloadsResult = await this.payloads.insertMany(payloadsWithExternalMeta)\n      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length)\n        throw new Error('MongoDBArchivist: Error inserting Payloads')\n    }\n    const boundWitnessesWithExternalMeta = await Promise.all(bw.map((value, index) => toDbRepresentation(value, index)))\n    if (boundWitnessesWithExternalMeta.length > 0) {\n      const boundWitnessesResult = await this.boundWitnesses.insertMany(boundWitnessesWithExternalMeta)\n      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length)\n        throw new Error('MongoDBArchivist: Error inserting BoundWitnesses')\n    }\n\n    return await PayloadBuilder.build([...boundWitnessesWithExternalMeta, ...payloadsWithExternalMeta].map(fromDbRepresentation))\n  }\n\n  protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithMeta<Payload>[]> {\n    // Sanitize inputs and set defaults\n    let {\n      limit, offset, order,\n    } = options ?? { limit: 10, order: 'desc' }\n\n    if (!limit) limit = 10\n    if (limit > 100) limit = 100\n\n    if (order != 'asc') order = 'desc'\n\n    let id: ObjectId | undefined\n    if (offset) {\n      const payload = await this.findOneByHash(offset)\n      // TODO: Should we throw an error if the requested payload is not found?\n      if (payload) id = payload._id\n    } else {\n      id = order === 'asc'\n      // If ascending, start from the beginning of time\n        ? ObjectId.createFromTime(0)\n        // If descending, start from now (plus a bit more in the future to ensure\n        // them most recent ObjectIds are included)\n        : ObjectId.createFromTime((Date.now() + 10_000) / 1000)\n    }\n    if (!id) return []\n\n    // Create aggregate criteria\n    const sort = order === 'asc' ? 1 : -1\n    // TODO: How to handle random component of ID across multiple collections\n    // to ensure we don't skip some payloads\n    const match = order === 'asc' ? { _id: { $gt: id } } : { _id: { $lt: id } }\n\n    // Run the aggregate query\n    const foundPayloads = await this.payloads.useCollection((collection) => {\n      return collection\n        .aggregate<PayloadWithMongoMeta>([\n          // Pre-filter payloads collection\n          { $match: match },\n          // Sort payloads by _id\n          { $sort: { _id: sort } },\n          // Limit payloads to the first N payloads\n          { $limit: limit },\n          // Combine with filtered boundWitnesses collection\n          {\n            $unionWith: {\n              coll: this.boundWitnessSdkConfig.collection,\n              pipeline: [\n                { $match: match }, // Pre-filter boundWitnesses\n                { $sort: { _id: sort } }, // Sort boundWitnesses by _id\n                { $limit: limit }, // Limit boundWitnesses to the first N boundWitnesses\n              ],\n            },\n          },\n          // Sort the combined result by _id\n          { $sort: { _id: sort } },\n          // Limit the final result to N documents\n          { $limit: limit },\n        ])\n        .maxTimeMS(this.aggregateTimeoutMs)\n        .toArray()\n    })\n\n    // Convert from DB representation to Payloads\n    return await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    await this.ensureIndexes()\n    return true\n  }\n}\n","import type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness, isQueryBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessWrapper, QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-wrapper'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\n\nexport const validByType = async (payloads: Payload[] = []) => {\n  const results: [BoundWitness[], Payload[]] = [[], []]\n  await Promise.all(\n    payloads.map(async (payload) => {\n      if (isBoundWitness(payload)) {\n        const wrapper = isQueryBoundWitness(payload) ? QueryBoundWitnessWrapper : BoundWitnessWrapper\n        const bw = wrapper.parse(payload)\n        if (await bw.getValid()) {\n          results[0].push(payload)\n        } else {\n          const errors = await bw.getErrors()\n          console.log(`validByType.Error: ${JSON.stringify(errors, null, 2)}`)\n        }\n      } else {\n        const payloadWrapper = PayloadWrapper.wrap(payload)\n        if (await payloadWrapper.getValid()) {\n          results[1].push(payload)\n        }\n      }\n      return\n    }),\n  )\n  return results\n}\n","export * from './Archivist.js'\nexport * from '@xyo-network/archivist-model-mongodb'\n"],"mappings":";;;;AAAA,SAASA,cAAc;AAEvB,SAASC,yBAAyB;AAElC,SAASC,4BAA4BC,gCAAgC;AACrE,SAASC,oCAAoC;AAC7C,SAASC,0BAA0B;AACnC,SAASC,sBAAsB;AAK/B,SAASC,sBAAsBC,0BAA0B;AACzD,SAASC,kBAAAA,uBAAsB;AAC/B,SAASC,gBAAgB;;;ACbzB,SAASC,gBAAgBC,2BAA2B;AACpD,SAASC,qBAAqBC,gCAAgC;AAE9D,SAASC,sBAAsB;AAExB,IAAMC,cAAc,8BAAOC,WAAsB,CAAA,MAAE;AACxD,QAAMC,UAAuC;IAAC,CAAA;IAAI,CAAA;;AAClD,QAAMC,QAAQC,IACZH,SAASI,IAAI,OAAOC,YAAAA;AAClB,QAAIC,eAAeD,OAAAA,GAAU;AAC3B,YAAME,UAAUC,oBAAoBH,OAAAA,IAAWI,2BAA2BC;AAC1E,YAAMC,KAAKJ,QAAQK,MAAMP,OAAAA;AACzB,UAAI,MAAMM,GAAGE,SAAQ,GAAI;AACvBZ,gBAAQ,CAAA,EAAGa,KAAKT,OAAAA;MAClB,OAAO;AACL,cAAMU,SAAS,MAAMJ,GAAGK,UAAS;AACjCC,gBAAQC,IAAI,sBAAsBC,KAAKC,UAAUL,QAAQ,MAAM,CAAA,CAAA,EAAI;MACrE;IACF,OAAO;AACL,YAAMM,iBAAiBC,eAAeC,KAAKlB,OAAAA;AAC3C,UAAI,MAAMgB,eAAeR,SAAQ,GAAI;AACnCZ,gBAAQ,CAAA,EAAGa,KAAKT,OAAAA;MAClB;IACF;AACA;EACF,CAAA,CAAA;AAEF,SAAOJ;AACT,GAvB2B;;;ADY3B,IAAMuB,uBAAuBC,mBAAmBC,iBAAAA;AAEzC,IAAMC,mBAAN,cAA+BH,qBAAAA;EApBtC,OAoBsCA;;;EACpC,OAAyBI,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAErCE,UAAoB;IAACC;IAA4BC;OAA6B,MAAMF;;;;;EAKnFG,qBAAqB;EAExC,MAAeC,OAAqC;AAClD,UAAMA,OAAO,MAAM,KAAKC,KAAK;MAAEC,OAAO;MAAGC,OAAO;IAAO,CAAA;AACvD,WAAOH,KAAK,CAAA,IAAKI,gBAAeC,KAAKL,KAAK,CAAA,CAAE,EAAEM,UAAUC;EAC1D;EAEA,MAAgBC,cAAcC,MAAY;AACxC,UAAMC,cAAe,MAAM,KAAKC,SAASC,QAAQ;MAAEC,QAAQJ;IAAK,CAAA;AAChE,QAAIC,aAAa;AACf,aAAOA;IACT,OAAO;AACL,YAAMI,SAAU,MAAM,KAAKC,eAAeH,QAAQ;QAAEC,QAAQJ;MAAK,CAAA;AACjE,UAAIK,QAAQ;AACV,eAAOA;MACT,OAAO;AACL,cAAMR,UAAW,MAAM,KAAKK,SAASC,QAAQ;UAAEI,OAAOP;QAAK,CAAA;AAC3D,YAAIH,SAAS;AACX,iBAAOA;QACT,OAAO;AACL,gBAAMW,KAAM,MAAM,KAAKF,eAAeH,QAAQ;YAAEI,OAAOP;UAAK,CAAA;AAC5D,iBAAOQ,MAAMV;QACf;MACF;IACF;EACF;EAEA,MAAyBW,WAAWC,QAA8C;AAChF,QAAIC,kBAAkB;SAAID;;AAE1B,UAAME,gBAAgB,MAAMC,QAAQC,IAAIH,gBAAgBI,IAAIX,CAAAA,WAAU,KAAKF,SAASC,QAAQ;MAAEC;IAAO,CAAA,CAAA,CAAA,GAAMY,OAAOC,MAAAA;AAClH,UAAMC,qBAAqB,IAAIC,IAAIP,aAAaG,IAAIlB,CAAAA,YAAWA,QAAQO,MAAM,CAAA;AAC7EO,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACkB,mBAAmBE,IAAIpB,IAAAA,CAAAA;AAEzE,UAAMqB,WAAW,MAAMR,QAAQC,IAAIH,gBAAgBI,IAAIX,CAAAA,WAAU,KAAKE,eAAeH,QAAQ;MAAEC;IAAO,CAAA,CAAA,CAAA,GAAMY,OAAOC,MAAAA;AACnH,UAAMK,gBAAgB,IAAIH,IAAIE,QAAQN,IAAIlB,CAAAA,YAAWA,QAAQO,MAAM,CAAA;AACnEO,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACsB,cAAcF,IAAIpB,IAAAA,CAAAA;AAEpE,UAAME,YAAY,MAAMW,QAAQC,IAAIH,gBAAgBI,IAAIR,CAAAA,UAAS,KAAKL,SAASC,QAAQ;MAAEI;IAAM,CAAA,CAAA,CAAA,GAAMS,OAAOC,MAAAA;AAC5G,UAAMM,iBAAiB,IAAIJ,IAAIjB,SAASa,IAAIlB,CAAAA,YAAWA,QAAQU,KAAK,CAAA;AACpEI,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACuB,eAAeH,IAAIpB,IAAAA,CAAAA;AAErE,UAAMwB,OAAO,MAAMX,QAAQC,IAAIH,gBAAgBI,IAAIR,CAAAA,UAAS,KAAKD,eAAeH,QAAQ;MAAEI;IAAM,CAAA,CAAA,CAAA,GAAMS,OAAOC,MAAAA;AAC7G,UAAMQ,YAAY,IAAIN,IAAIK,IAAIT,IAAIlB,CAAAA,YAAWA,QAAQU,KAAK,CAAA;AAC1DI,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACyB,UAAUL,IAAIpB,IAAAA,CAAAA;AAEhE,UAAM0B,gBAAgB;SAAId;SAAiBS;SAAYnB;SAAasB;;AACpE,UAAMG,SAAS,MAAMC,eAAeC,MAAMH,cAAcX,IAAIe,oBAAAA,CAAAA;AAE5D,WAAOH;EACT;EAEA,MAAyBI,cAAc7B,UAAmD;AACxF,UAAM,CAACM,IAAIwB,CAAAA,IAAK,MAAMC,YAAY/B,QAAAA;AAClC,UAAMgC,2BAA2B,MAAMrB,QAAQC,IAAIkB,EAAEjB,IAAI,CAACoB,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AACrG,QAAIF,yBAAyBI,SAAS,GAAG;AACvC,YAAMC,iBAAiB,MAAM,KAAKrC,SAASsC,WAAWN,wBAAAA;AACtD,UAAI,CAACK,eAAeE,gBAAgBF,eAAeG,kBAAkBR,yBAAyBI,OAC5F,OAAM,IAAIK,MAAM,4CAAA;IACpB;AACA,UAAMC,iCAAiC,MAAM/B,QAAQC,IAAIN,GAAGO,IAAI,CAACoB,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AAC5G,QAAIQ,+BAA+BN,SAAS,GAAG;AAC7C,YAAMO,uBAAuB,MAAM,KAAKvC,eAAekC,WAAWI,8BAAAA;AAClE,UAAI,CAACC,qBAAqBJ,gBAAgBI,qBAAqBH,kBAAkBE,+BAA+BN,OAC9G,OAAM,IAAIK,MAAM,kDAAA;IACpB;AAEA,WAAO,MAAMf,eAAeC,MAAM;SAAIe;SAAmCV;MAA0BnB,IAAIe,oBAAAA,CAAAA;EACzG;EAEA,MAAyBgB,YAAYC,SAA8D;AAEjG,QAAI,EACFtD,OAAOuD,QAAQtD,MAAK,IAClBqD,WAAW;MAAEtD,OAAO;MAAIC,OAAO;IAAO;AAE1C,QAAI,CAACD,MAAOA,SAAQ;AACpB,QAAIA,QAAQ,IAAKA,SAAQ;AAEzB,QAAIC,SAAS,MAAOA,SAAQ;AAE5B,QAAIuD;AACJ,QAAID,QAAQ;AACV,YAAMnD,UAAU,MAAM,KAAKE,cAAciD,MAAAA;AAEzC,UAAInD,QAASoD,MAAKpD,QAAQqD;IAC5B,OAAO;AACLD,WAAKvD,UAAU,QAEXyD,SAASC,eAAe,CAAA,IAGxBD,SAASC,gBAAgBC,KAAKC,IAAG,IAAK,OAAU,GAAA;IACtD;AACA,QAAI,CAACL,GAAI,QAAO,CAAA;AAGhB,UAAMM,OAAO7D,UAAU,QAAQ,IAAI;AAGnC,UAAM8D,QAAQ9D,UAAU,QAAQ;MAAEwD,KAAK;QAAEO,KAAKR;MAAG;IAAE,IAAI;MAAEC,KAAK;QAAEQ,KAAKT;MAAG;IAAE;AAG1E,UAAMvB,gBAAgB,MAAM,KAAKxB,SAASyD,cAAc,CAACC,eAAAA;AACvD,aAAOA,WACJC,UAAgC;;QAE/B;UAAEC,QAAQN;QAAM;;QAEhB;UAAEO,OAAO;YAAEb,KAAKK;UAAK;QAAE;;QAEvB;UAAES,QAAQvE;QAAM;;QAEhB;UACEwE,YAAY;YACVC,MAAM,KAAKC,sBAAsBP;YACjCQ,UAAU;cACR;gBAAEN,QAAQN;cAAM;cAChB;gBAAEO,OAAO;kBAAEb,KAAKK;gBAAK;cAAE;cACvB;gBAAES,QAAQvE;cAAM;;UAEpB;QACF;;QAEA;UAAEsE,OAAO;YAAEb,KAAKK;UAAK;QAAE;;QAEvB;UAAES,QAAQvE;QAAM;OACjB,EACA4E,UAAU,KAAK/E,kBAAkB,EACjCgF,QAAO;IACZ,CAAA;AAGA,WAAO,MAAM1C,eAAeC,MAAMH,cAAcX,IAAIe,oBAAAA,CAAAA;EACtD;EAEA,MAAyByC,eAAe;AACtC,UAAM,MAAMA,aAAAA;AACZ,UAAM,KAAKC,cAAa;AACxB,WAAO;EACT;AACF;;;AEzKA,cAAc;","names":["exists","AbstractArchivist","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","MongoDBArchivistConfigSchema","MongoDBModuleMixin","PayloadBuilder","fromDbRepresentation","toDbRepresentation","PayloadWrapper","ObjectId","isBoundWitness","isQueryBoundWitness","BoundWitnessWrapper","QueryBoundWitnessWrapper","PayloadWrapper","validByType","payloads","results","Promise","all","map","payload","isBoundWitness","wrapper","isQueryBoundWitness","QueryBoundWitnessWrapper","BoundWitnessWrapper","bw","parse","getValid","push","errors","getErrors","console","log","JSON","stringify","payloadWrapper","PayloadWrapper","wrap","MongoDBArchivistBase","MongoDBModuleMixin","AbstractArchivist","MongoDBArchivist","configSchemas","MongoDBArchivistConfigSchema","defaultConfigSchema","queries","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","aggregateTimeoutMs","head","next","limit","order","PayloadWrapper","wrap","payload","undefined","findOneByHash","hash","dataPayload","payloads","findOne","_$hash","dataBw","boundWitnesses","_hash","bw","getHandler","hashes","remainingHashes","dataPayloads","Promise","all","map","filter","exists","dataPayloadsHashes","Set","has","dataBws","dataBwsHashes","payloadsHashes","bws","bwsHashes","foundPayloads","result","PayloadBuilder","build","fromDbRepresentation","insertHandler","p","validByType","payloadsWithExternalMeta","value","index","toDbRepresentation","length","payloadsResult","insertMany","acknowledged","insertedCount","Error","boundWitnessesWithExternalMeta","boundWitnessesResult","nextHandler","options","offset","id","_id","ObjectId","createFromTime","Date","now","sort","match","$gt","$lt","useCollection","collection","aggregate","$match","$sort","$limit","$unionWith","coll","boundWitnessSdkConfig","pipeline","maxTimeMS","toArray","startHandler","ensureIndexes"]}

package/dist/neutral/Archivist.d.ts

@@ -1,5 +1,6 @@
 import type { Hash } from '@xylabs/hex';
 import { AbstractArchivist } from '@xyo-network/archivist-abstract';
+import type { ArchivistNextOptions } from '@xyo-network/archivist-model';
 import type { Payload, Schema, WithMeta } from '@xyo-network/payload-model';
 import type { PayloadWithMongoMeta } from '@xyo-network/payload-mongodb';
 declare const MongoDBArchivistBase: (abstract new (...args: any[]) => {
@@ -67,9 +68,15 @@ export declare class MongoDBArchivist extends MongoDBArchivistBase {
     static readonly configSchemas: Schema[];
     static readonly defaultConfigSchema: Schema;
     readonly queries: string[];
+    /**
+     * The amount of time to allow the aggregate query to execute
+     */
+    protected readonly aggregateTimeoutMs = 10000;
     head(): Promise<Payload | undefined>;
+    protected findOneByHash(hash: Hash): Promise<import("mongodb").WithId<PayloadWithMongoMeta> | undefined>;
     protected getHandler(hashes: Hash[]): Promise<WithMeta<Payload>[]>;
     protected insertHandler(payloads: Payload[]): Promise<WithMeta<Payload>[]>;
+    protected nextHandler(options?: ArchivistNextOptions): Promise<WithMeta<Payload>[]>;
     protected startHandler(): Promise<boolean>;
 }
 export {};

package/dist/neutral/Archivist.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Archivist.d.ts","sourceRoot":"","sources":["../../src/Archivist.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AAKnE,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAAE,QAAQ,EAC1B,MAAM,4BAA4B,CAAA;AACnC,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAA;AAMxE,QAAA,MAAM,oBAAoB;;;;;8BAgCoD,8BAA8B;;;;oBAOtE,aACnC;;+CA/B0E,+BACnE;0CAGQ,+BAA+B;uBACtB,mCAEJ;;;;;;;qBAwB2D,2BAA2B;;yBA7CrF,4BAA4B;;mCACG,4BAEnC;;uBAE6B,4BAExC,wBAAwB,4BACjC,8BAA8B,2BAA2B,iCAAiC,2BACjF,8BAA8B,4BAA4B,wBAE/D,4BAA4B,8BAA8B,2BAE7D,iCAAiC,2BAC7B;iCAAsD,CAAC,iBAAiB,+BACnE;4BAGR,CAAD,iBAAiB,+BAA+B;2BACtB,mCAEJ;;;;;;;8CAE+B,2BAA2B;;wCAEV,+BAA+B,6CAClF,+BAA+B;0BACV,8BAA8B;mCAErC,+BAA+B,6CAA6C,+BACzF;;+BAiBP,iBAAiB;;6BACA,iCAAiC,6BAA6B,iCAC3E,0CAA0C,2BACxD,wBACkB,2BAA2B,kCAAkC,CAAC,SAAS,4BAC7F,wBAAwB,CAAC,mBAAmB,iBACpC,oBAAoB,2BAA2B;iCACjD,iCAAiC,6BAA6B,iCAAiC,4CAClF,2BAA2B,wBAAwB,2BAGrE,kCAAkC,CAAC,SAAS,4BAA4B,wBAAwB,CAAC,qBAGxF,iBAAiB;yBAEvB,iBAAa;wBAIN,iBAAiB;sBAA6C,2BAA2B;4CAAuE,2BAA2B,0CAA0C,2BAA2B;yCAA8E,2BAA2B,oEAAoE,2BAA2B;+CAAsG,2BAA2B,oEAAoE,2BAA2B;2CAAkG,2BAA2B,0CAA0C,2BAA2B;wCAA+E,2BAA2B,mFAAmF,4BAA4B,uBAAuB,2BAA2B;4BAA2E,4BAA4B;uCAAkF,2BAA2B,mFAAmF,4BAA4B,uBAAuB,2BAA2B,gDAAgD,4BAA4B;2BAAuD,4BAA4B,4BAA4B,4BAA4B;yCAAqE,2BAA2B,mEAAmE,4BAA4B,uBAAuB,2BAA2B,gDAAgD,4BAA4B;;;4BA9Dv8D,CAAA;AAElE,qBAAa,gBAAiB,SAAQ,oBAAoB;IACxD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAyD;IACzG,gBAAyB,mBAAmB,EAAE,MAAM,CAA+B;IAEnF,SAAkB,OAAO,EAAE,MAAM,EAAE,CAAiD;IAErE,IAAI,IAAI,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC;cAK1B,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAyBxD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAkBhE,YAAY;CAKtC"}
+{"version":3,"file":"Archivist.d.ts","sourceRoot":"","sources":["../../src/Archivist.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAA;AAKxE,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAAE,QAAQ,EAC1B,MAAM,4BAA4B,CAAA;AACnC,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAA;AAOxE,QAAA,MAAM,oBAAoB;;;;;8BA6CsC,8BAA8B;;;;oBAMzF,aAAa;;+CA7CqE,+BAEnF;0CAGQ,+BAA8B;uBAGtC,mCAA+B;;;;;;;qBAuCnB,2BAA2B;;yBA7DT,4BACR;;mCACkB,4BAA4B;;uBACb,4BAClC,wBAEZ,4BAA4B,8BAEhB,2BAA2B,iCAEnC,2BAA2B,8BAC/B,4BAA4B,wBAAwB,4BAA4B,8BAC9E,2BAA2B,iCAAiC,2BAE5D;iCAAsD,CAAC,iBAAiB,+BAEnF;4BAED,CAAH,iBACY,+BAA8B;2BAGtC,mCAA+B;;;;;;;8CAMd,2BAErB;;wCAC8D,+BAC5C,6CAGV,+BAA6B;0BAE3B,8BACE;mCAC0B,+BAA+B,6CAE7D,+BAKC;;+BAeoG,iBACxG;;6BACqB,iCAAiC,6BAE/C,iCAAiC,0CAA0C,2BAA2B,wBAAwB,2BAC1H,kCAAkC,CAAC,SAAS,4BAC3D,wBAAwB,CAAC,mBAAmB,iBAAiB,oBAAoB,2BAC7E;iCAGgC,iCAAiC,6BACjE,iCAAiC,4CACH,2BAA0B,wBAAwB,2BAA2B,kCAChF,CAAC,SAAS,4BACnB,wBAAwB,CAAC,qBAAqB,iBACrE;yBAAgD,iBAAiB;wBACzD,iBAAa;sBAA6C,2BAEjD;4CAAuE,2BAA2B,0CAC/E,2BAC1B;yCAA8E,2BAC7E,oEAAoE,2BAA2B;+CAItG,2BAAsB,oEAAoE,2BAA2B;2CAGrC,2BAC5E,0CAEF,2BAAqB;wCAIvB,2BAAyB,mFAKxB,4BACU,uBAAuB,2BAC5B;4BACJ,4BAA4B;uCAGM,2BAC7B,mFAC+C,4BACvD,uBAAuB,2BACzB,gDAAgD,4BAE1C;2BAGF,4BAA4B,4BACb,4BAA2B;yCACT,2BAClB,mEAEb,4BACJ,uBAAuB,2BAA2B,gDAE7C,4BAAmB;;;4BApHuC,CAAA;AAElE,qBAAa,gBAAiB,SAAQ,oBAAoB;IACxD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAyD;IACzG,gBAAyB,mBAAmB,EAAE,MAAM,CAA+B;IAEnF,SAAkB,OAAO,EAAE,MAAM,EAAE,CAA2E;IAE9G;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,kBAAkB,SAAS;IAE/B,IAAI,IAAI,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC;cAKnC,aAAa,CAAC,IAAI,EAAE,IAAI;cAoBf,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAyBxD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAkBhE,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAkEzE,YAAY;CAKtC"}

package/dist/neutral/index.mjs

@@ -4,12 +4,13 @@ var __name = (target, value) => __defProp(target, "name", { value, configurable:
 // src/Archivist.ts
 import { exists } from "@xylabs/exists";
 import { AbstractArchivist } from "@xyo-network/archivist-abstract";
-import { ArchivistInsertQuerySchema } from "@xyo-network/archivist-model";
+import { ArchivistInsertQuerySchema, ArchivistNextQuerySchema } from "@xyo-network/archivist-model";
 import { MongoDBArchivistConfigSchema } from "@xyo-network/archivist-model-mongodb";
 import { MongoDBModuleMixin } from "@xyo-network/module-abstract-mongodb";
 import { PayloadBuilder } from "@xyo-network/payload-builder";
 import { fromDbRepresentation, toDbRepresentation } from "@xyo-network/payload-mongodb";
 import { PayloadWrapper as PayloadWrapper2 } from "@xyo-network/payload-wrapper";
+import { ObjectId } from "mongodb";
 
 // src/lib/validByType.ts
 import { isBoundWitness, isQueryBoundWitness } from "@xyo-network/boundwitness-model";
@@ -25,7 +26,7 @@ var validByType = /* @__PURE__ */ __name(async (payloads = []) => {
       const wrapper = isQueryBoundWitness(payload) ? QueryBoundWitnessWrapper : BoundWitnessWrapper;
       const bw = wrapper.parse(payload);
       if (await bw.getValid()) {
-        results[0].push(bw.payload);
+        results[0].push(payload);
       } else {
         const errors = await bw.getErrors();
         console.log(`validByType.Error: ${JSON.stringify(errors, null, 2)}`);
@@ -33,7 +34,7 @@ var validByType = /* @__PURE__ */ __name(async (payloads = []) => {
     } else {
       const payloadWrapper = PayloadWrapper.wrap(payload);
       if (await payloadWrapper.getValid()) {
-        results[1].push(payloadWrapper.payload);
+        results[1].push(payload);
       }
     }
     return;
@@ -54,14 +55,47 @@ var MongoDBArchivist = class extends MongoDBArchivistBase {
   static defaultConfigSchema = MongoDBArchivistConfigSchema;
   queries = [
     ArchivistInsertQuerySchema,
+    ArchivistNextQuerySchema,
     ...super.queries
   ];
+  /**
+  * The amount of time to allow the aggregate query to execute
+  */
+  aggregateTimeoutMs = 1e4;
   async head() {
-    const head = await (await this.payloads.find({})).sort({
-      _timestamp: -1
-    }).limit(1).toArray();
+    const head = await this.next({
+      limit: 1,
+      order: "desc"
+    });
     return head[0] ? PayloadWrapper2.wrap(head[0]).payload : void 0;
   }
+  async findOneByHash(hash) {
+    const dataPayload = await this.payloads.findOne({
+      _$hash: hash
+    });
+    if (dataPayload) {
+      return dataPayload;
+    } else {
+      const dataBw = await this.boundWitnesses.findOne({
+        _$hash: hash
+      });
+      if (dataBw) {
+        return dataBw;
+      } else {
+        const payload = await this.payloads.findOne({
+          _hash: hash
+        });
+        if (payload) {
+          return payload;
+        } else {
+          const bw = await this.boundWitnesses.findOne({
+            _hash: hash
+          });
+          return bw ?? void 0;
+        }
+      }
+    }
+  }
   async getHandler(hashes) {
     let remainingHashes = [
       ...hashes
@@ -100,17 +134,92 @@ var MongoDBArchivist = class extends MongoDBArchivistBase {
     const payloadsWithExternalMeta = await Promise.all(p.map((value, index) => toDbRepresentation(value, index)));
     if (payloadsWithExternalMeta.length > 0) {
       const payloadsResult = await this.payloads.insertMany(payloadsWithExternalMeta);
-      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length) throw new Error("MongoDBDeterministicArchivist: Error inserting Payloads");
+      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length) throw new Error("MongoDBArchivist: Error inserting Payloads");
     }
     const boundWitnessesWithExternalMeta = await Promise.all(bw.map((value, index) => toDbRepresentation(value, index)));
     if (boundWitnessesWithExternalMeta.length > 0) {
       const boundWitnessesResult = await this.boundWitnesses.insertMany(boundWitnessesWithExternalMeta);
-      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length) throw new Error("MongoDBDeterministicArchivist: Error inserting BoundWitnesses");
+      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length) throw new Error("MongoDBArchivist: Error inserting BoundWitnesses");
     }
     return await PayloadBuilder.build([
       ...boundWitnessesWithExternalMeta,
       ...payloadsWithExternalMeta
-    ]);
+    ].map(fromDbRepresentation));
+  }
+  async nextHandler(options) {
+    let { limit, offset, order } = options ?? {
+      limit: 10,
+      order: "desc"
+    };
+    if (!limit) limit = 10;
+    if (limit > 100) limit = 100;
+    if (order != "asc") order = "desc";
+    let id;
+    if (offset) {
+      const payload = await this.findOneByHash(offset);
+      if (payload) id = payload._id;
+    } else {
+      id = order === "asc" ? ObjectId.createFromTime(0) : ObjectId.createFromTime((Date.now() + 1e4) / 1e3);
+    }
+    if (!id) return [];
+    const sort = order === "asc" ? 1 : -1;
+    const match = order === "asc" ? {
+      _id: {
+        $gt: id
+      }
+    } : {
+      _id: {
+        $lt: id
+      }
+    };
+    const foundPayloads = await this.payloads.useCollection((collection) => {
+      return collection.aggregate([
+        // Pre-filter payloads collection
+        {
+          $match: match
+        },
+        // Sort payloads by _id
+        {
+          $sort: {
+            _id: sort
+          }
+        },
+        // Limit payloads to the first N payloads
+        {
+          $limit: limit
+        },
+        // Combine with filtered boundWitnesses collection
+        {
+          $unionWith: {
+            coll: this.boundWitnessSdkConfig.collection,
+            pipeline: [
+              {
+                $match: match
+              },
+              {
+                $sort: {
+                  _id: sort
+                }
+              },
+              {
+                $limit: limit
+              }
+            ]
+          }
+        },
+        // Sort the combined result by _id
+        {
+          $sort: {
+            _id: sort
+          }
+        },
+        // Limit the final result to N documents
+        {
+          $limit: limit
+        }
+      ]).maxTimeMS(this.aggregateTimeoutMs).toArray();
+    });
+    return await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation));
   }
   async startHandler() {
     await super.startHandler();

package/dist/neutral/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Archivist.ts","../../src/lib/validByType.ts","../../src/index.ts"],"sourcesContent":["import { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport { ArchivistInsertQuerySchema } from '@xyo-network/archivist-model'\nimport { MongoDBArchivistConfigSchema } from '@xyo-network/archivist-model-mongodb'\nimport { MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema, WithMeta,\n} from '@xyo-network/payload-model'\nimport type { PayloadWithMongoMeta } from '@xyo-network/payload-mongodb'\nimport { fromDbRepresentation, toDbRepresentation } from '@xyo-network/payload-mongodb'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\n\nimport { validByType } from './lib/index.js'\n\nconst MongoDBArchivistBase = MongoDBModuleMixin(AbstractArchivist)\n\nexport class MongoDBArchivist extends MongoDBArchivistBase {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, MongoDBArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = MongoDBArchivistConfigSchema\n\n  override readonly queries: string[] = [ArchivistInsertQuerySchema, ...super.queries]\n\n  override async head(): Promise<Payload | undefined> {\n    const head = await (await this.payloads.find({})).sort({ _timestamp: -1 }).limit(1).toArray()\n    return head[0] ? PayloadWrapper.wrap(head[0]).payload : undefined\n  }\n\n  protected override async getHandler(hashes: Hash[]): Promise<WithMeta<Payload>[]> {\n    let remainingHashes = [...hashes]\n\n    const dataPayloads = (await Promise.all(remainingHashes.map(_$hash => this.payloads.findOne({ _$hash })))).filter(exists)\n    const dataPayloadsHashes = new Set(dataPayloads.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataPayloadsHashes.has(hash))\n\n    const dataBws = (await Promise.all(remainingHashes.map(_$hash => this.boundWitnesses.findOne({ _$hash })))).filter(exists)\n    const dataBwsHashes = new Set(dataBws.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataBwsHashes.has(hash))\n\n    const payloads = (await Promise.all(remainingHashes.map(_hash => this.payloads.findOne({ _hash })))).filter(exists)\n    const payloadsHashes = new Set(payloads.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !payloadsHashes.has(hash))\n\n    const bws = (await Promise.all(remainingHashes.map(_hash => this.boundWitnesses.findOne({ _hash })))).filter(exists)\n    const bwsHashes = new Set(bws.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !bwsHashes.has(hash))\n\n    const foundPayloads = [...dataPayloads, ...dataBws, ...payloads, ...bws] as PayloadWithMongoMeta<Payload & { _$hash: Hash; _$meta?: unknown }>[]\n    const result = await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))\n    // console.log(`getHandler: ${JSON.stringify(hashes, null, 2)}:${JSON.stringify(result, null, 2)}`)\n    return result\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<WithMeta<Payload>[]> {\n    const [bw, p] = await validByType(payloads)\n    const payloadsWithExternalMeta = await Promise.all(p.map((value, index) => toDbRepresentation(value, index)))\n    if (payloadsWithExternalMeta.length > 0) {\n      const payloadsResult = await this.payloads.insertMany(payloadsWithExternalMeta)\n      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length)\n        throw new Error('MongoDBDeterministicArchivist: Error inserting Payloads')\n    }\n    const boundWitnessesWithExternalMeta = await Promise.all(bw.map((value, index) => toDbRepresentation(value, index)))\n    if (boundWitnessesWithExternalMeta.length > 0) {\n      const boundWitnessesResult = await this.boundWitnesses.insertMany(boundWitnessesWithExternalMeta)\n      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length)\n        throw new Error('MongoDBDeterministicArchivist: Error inserting BoundWitnesses')\n    }\n\n    return await PayloadBuilder.build([...boundWitnessesWithExternalMeta, ...payloadsWithExternalMeta])\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    await this.ensureIndexes()\n    return true\n  }\n}\n","import type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness, isQueryBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessWrapper, QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-wrapper'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\n\nexport const validByType = async (payloads: Payload[] = []) => {\n  const results: [BoundWitness[], Payload[]] = [[], []]\n  await Promise.all(\n    payloads.map(async (payload) => {\n      if (isBoundWitness(payload)) {\n        const wrapper = isQueryBoundWitness(payload) ? QueryBoundWitnessWrapper : BoundWitnessWrapper\n        const bw = wrapper.parse(payload)\n        if (await bw.getValid()) {\n          results[0].push(bw.payload)\n        } else {\n          const errors = await bw.getErrors()\n          console.log(`validByType.Error: ${JSON.stringify(errors, null, 2)}`)\n        }\n      } else {\n        const payloadWrapper = PayloadWrapper.wrap(payload)\n        if (await payloadWrapper.getValid()) {\n          results[1].push(payloadWrapper.payload)\n        }\n      }\n      return\n    }),\n  )\n  return results\n}\n","export * from './Archivist.js'\nexport * from '@xyo-network/archivist-model-mongodb'\n"],"mappings":";;;;AAAA,SAASA,cAAc;AAEvB,SAASC,yBAAyB;AAClC,SAASC,kCAAkC;AAC3C,SAASC,oCAAoC;AAC7C,SAASC,0BAA0B;AACnC,SAASC,sBAAsB;AAK/B,SAASC,sBAAsBC,0BAA0B;AACzD,SAASC,kBAAAA,uBAAsB;;;ACX/B,SAASC,gBAAgBC,2BAA2B;AACpD,SAASC,qBAAqBC,gCAAgC;AAE9D,SAASC,sBAAsB;AAExB,IAAMC,cAAc,8BAAOC,WAAsB,CAAA,MAAE;AACxD,QAAMC,UAAuC;IAAC,CAAA;IAAI,CAAA;;AAClD,QAAMC,QAAQC,IACZH,SAASI,IAAI,OAAOC,YAAAA;AAClB,QAAIC,eAAeD,OAAAA,GAAU;AAC3B,YAAME,UAAUC,oBAAoBH,OAAAA,IAAWI,2BAA2BC;AAC1E,YAAMC,KAAKJ,QAAQK,MAAMP,OAAAA;AACzB,UAAI,MAAMM,GAAGE,SAAQ,GAAI;AACvBZ,gBAAQ,CAAA,EAAGa,KAAKH,GAAGN,OAAO;MAC5B,OAAO;AACL,cAAMU,SAAS,MAAMJ,GAAGK,UAAS;AACjCC,gBAAQC,IAAI,sBAAsBC,KAAKC,UAAUL,QAAQ,MAAM,CAAA,CAAA,EAAI;MACrE;IACF,OAAO;AACL,YAAMM,iBAAiBC,eAAeC,KAAKlB,OAAAA;AAC3C,UAAI,MAAMgB,eAAeR,SAAQ,GAAI;AACnCZ,gBAAQ,CAAA,EAAGa,KAAKO,eAAehB,OAAO;MACxC;IACF;AACA;EACF,CAAA,CAAA;AAEF,SAAOJ;AACT,GAvB2B;;;ADU3B,IAAMuB,uBAAuBC,mBAAmBC,iBAAAA;AAEzC,IAAMC,mBAAN,cAA+BH,qBAAAA;EAlBtC,OAkBsCA;;;EACpC,OAAyBI,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAErCE,UAAoB;IAACC;OAA+B,MAAMD;;EAE5E,MAAeE,OAAqC;AAClD,UAAMA,OAAO,OAAO,MAAM,KAAKC,SAASC,KAAK,CAAC,CAAA,GAAIC,KAAK;MAAEC,YAAY;IAAG,CAAA,EAAGC,MAAM,CAAA,EAAGC,QAAO;AAC3F,WAAON,KAAK,CAAA,IAAKO,gBAAeC,KAAKR,KAAK,CAAA,CAAE,EAAES,UAAUC;EAC1D;EAEA,MAAyBC,WAAWC,QAA8C;AAChF,QAAIC,kBAAkB;SAAID;;AAE1B,UAAME,gBAAgB,MAAMC,QAAQC,IAAIH,gBAAgBI,IAAIC,CAAAA,WAAU,KAAKjB,SAASkB,QAAQ;MAAED;IAAO,CAAA,CAAA,CAAA,GAAME,OAAOC,MAAAA;AAClH,UAAMC,qBAAqB,IAAIC,IAAIT,aAAaG,IAAIR,CAAAA,YAAWA,QAAQS,MAAM,CAAA;AAC7EL,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACF,mBAAmBG,IAAID,IAAAA,CAAAA;AAEzE,UAAME,WAAW,MAAMX,QAAQC,IAAIH,gBAAgBI,IAAIC,CAAAA,WAAU,KAAKS,eAAeR,QAAQ;MAAED;IAAO,CAAA,CAAA,CAAA,GAAME,OAAOC,MAAAA;AACnH,UAAMO,gBAAgB,IAAIL,IAAIG,QAAQT,IAAIR,CAAAA,YAAWA,QAAQS,MAAM,CAAA;AACnEL,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACI,cAAcH,IAAID,IAAAA,CAAAA;AAEpE,UAAMvB,YAAY,MAAMc,QAAQC,IAAIH,gBAAgBI,IAAIY,CAAAA,UAAS,KAAK5B,SAASkB,QAAQ;MAAEU;IAAM,CAAA,CAAA,CAAA,GAAMT,OAAOC,MAAAA;AAC5G,UAAMS,iBAAiB,IAAIP,IAAItB,SAASgB,IAAIR,CAAAA,YAAWA,QAAQoB,KAAK,CAAA;AACpEhB,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACM,eAAeL,IAAID,IAAAA,CAAAA;AAErE,UAAMO,OAAO,MAAMhB,QAAQC,IAAIH,gBAAgBI,IAAIY,CAAAA,UAAS,KAAKF,eAAeR,QAAQ;MAAEU;IAAM,CAAA,CAAA,CAAA,GAAMT,OAAOC,MAAAA;AAC7G,UAAMW,YAAY,IAAIT,IAAIQ,IAAId,IAAIR,CAAAA,YAAWA,QAAQoB,KAAK,CAAA;AAC1DhB,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACQ,UAAUP,IAAID,IAAAA,CAAAA;AAEhE,UAAMS,gBAAgB;SAAInB;SAAiBY;SAAYzB;SAAa8B;;AACpE,UAAMG,SAAS,MAAMC,eAAeC,MAAMH,cAAchB,IAAIoB,oBAAAA,CAAAA;AAE5D,WAAOH;EACT;EAEA,MAAyBI,cAAcrC,UAAmD;AACxF,UAAM,CAACsC,IAAIC,CAAAA,IAAK,MAAMC,YAAYxC,QAAAA;AAClC,UAAMyC,2BAA2B,MAAM3B,QAAQC,IAAIwB,EAAEvB,IAAI,CAAC0B,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AACrG,QAAIF,yBAAyBI,SAAS,GAAG;AACvC,YAAMC,iBAAiB,MAAM,KAAK9C,SAAS+C,WAAWN,wBAAAA;AACtD,UAAI,CAACK,eAAeE,gBAAgBF,eAAeG,kBAAkBR,yBAAyBI,OAC5F,OAAM,IAAIK,MAAM,yDAAA;IACpB;AACA,UAAMC,iCAAiC,MAAMrC,QAAQC,IAAIuB,GAAGtB,IAAI,CAAC0B,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AAC5G,QAAIQ,+BAA+BN,SAAS,GAAG;AAC7C,YAAMO,uBAAuB,MAAM,KAAK1B,eAAeqB,WAAWI,8BAAAA;AAClE,UAAI,CAACC,qBAAqBJ,gBAAgBI,qBAAqBH,kBAAkBE,+BAA+BN,OAC9G,OAAM,IAAIK,MAAM,+DAAA;IACpB;AAEA,WAAO,MAAMhB,eAAeC,MAAM;SAAIgB;SAAmCV;KAAyB;EACpG;EAEA,MAAyBY,eAAe;AACtC,UAAM,MAAMA,aAAAA;AACZ,UAAM,KAAKC,cAAa;AACxB,WAAO;EACT;AACF;;;AE5EA,cAAc;","names":["exists","AbstractArchivist","ArchivistInsertQuerySchema","MongoDBArchivistConfigSchema","MongoDBModuleMixin","PayloadBuilder","fromDbRepresentation","toDbRepresentation","PayloadWrapper","isBoundWitness","isQueryBoundWitness","BoundWitnessWrapper","QueryBoundWitnessWrapper","PayloadWrapper","validByType","payloads","results","Promise","all","map","payload","isBoundWitness","wrapper","isQueryBoundWitness","QueryBoundWitnessWrapper","BoundWitnessWrapper","bw","parse","getValid","push","errors","getErrors","console","log","JSON","stringify","payloadWrapper","PayloadWrapper","wrap","MongoDBArchivistBase","MongoDBModuleMixin","AbstractArchivist","MongoDBArchivist","configSchemas","MongoDBArchivistConfigSchema","defaultConfigSchema","queries","ArchivistInsertQuerySchema","head","payloads","find","sort","_timestamp","limit","toArray","PayloadWrapper","wrap","payload","undefined","getHandler","hashes","remainingHashes","dataPayloads","Promise","all","map","_$hash","findOne","filter","exists","dataPayloadsHashes","Set","hash","has","dataBws","boundWitnesses","dataBwsHashes","_hash","payloadsHashes","bws","bwsHashes","foundPayloads","result","PayloadBuilder","build","fromDbRepresentation","insertHandler","bw","p","validByType","payloadsWithExternalMeta","value","index","toDbRepresentation","length","payloadsResult","insertMany","acknowledged","insertedCount","Error","boundWitnessesWithExternalMeta","boundWitnessesResult","startHandler","ensureIndexes"]}
+{"version":3,"sources":["../../src/Archivist.ts","../../src/lib/validByType.ts","../../src/index.ts"],"sourcesContent":["import { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport type { ArchivistNextOptions } from '@xyo-network/archivist-model'\nimport { ArchivistInsertQuerySchema, ArchivistNextQuerySchema } from '@xyo-network/archivist-model'\nimport { MongoDBArchivistConfigSchema } from '@xyo-network/archivist-model-mongodb'\nimport { MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema, WithMeta,\n} from '@xyo-network/payload-model'\nimport type { PayloadWithMongoMeta } from '@xyo-network/payload-mongodb'\nimport { fromDbRepresentation, toDbRepresentation } from '@xyo-network/payload-mongodb'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\nimport { ObjectId } from 'mongodb'\n\nimport { validByType } from './lib/index.js'\n\nconst MongoDBArchivistBase = MongoDBModuleMixin(AbstractArchivist)\n\nexport class MongoDBArchivist extends MongoDBArchivistBase {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, MongoDBArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = MongoDBArchivistConfigSchema\n\n  override readonly queries: string[] = [ArchivistInsertQuerySchema, ArchivistNextQuerySchema, ...super.queries]\n\n  /**\n   * The amount of time to allow the aggregate query to execute\n   */\n  protected readonly aggregateTimeoutMs = 10_000\n\n  override async head(): Promise<Payload | undefined> {\n    const head = await this.next({ limit: 1, order: 'desc' })\n    return head[0] ? PayloadWrapper.wrap(head[0]).payload : undefined\n  }\n\n  protected async findOneByHash(hash: Hash) {\n    const dataPayload = (await this.payloads.findOne({ _$hash: hash }))\n    if (dataPayload) {\n      return dataPayload\n    } else {\n      const dataBw = (await this.boundWitnesses.findOne({ _$hash: hash }))\n      if (dataBw) {\n        return dataBw\n      } else {\n        const payload = (await this.payloads.findOne({ _hash: hash }))\n        if (payload) {\n          return payload\n        } else {\n          const bw = (await this.boundWitnesses.findOne({ _hash: hash }))\n          return bw ?? undefined\n        }\n      }\n    }\n  }\n\n  protected override async getHandler(hashes: Hash[]): Promise<WithMeta<Payload>[]> {\n    let remainingHashes = [...hashes]\n\n    const dataPayloads = (await Promise.all(remainingHashes.map(_$hash => this.payloads.findOne({ _$hash })))).filter(exists)\n    const dataPayloadsHashes = new Set(dataPayloads.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataPayloadsHashes.has(hash))\n\n    const dataBws = (await Promise.all(remainingHashes.map(_$hash => this.boundWitnesses.findOne({ _$hash })))).filter(exists)\n    const dataBwsHashes = new Set(dataBws.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataBwsHashes.has(hash))\n\n    const payloads = (await Promise.all(remainingHashes.map(_hash => this.payloads.findOne({ _hash })))).filter(exists)\n    const payloadsHashes = new Set(payloads.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !payloadsHashes.has(hash))\n\n    const bws = (await Promise.all(remainingHashes.map(_hash => this.boundWitnesses.findOne({ _hash })))).filter(exists)\n    const bwsHashes = new Set(bws.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !bwsHashes.has(hash))\n\n    const foundPayloads = [...dataPayloads, ...dataBws, ...payloads, ...bws] as PayloadWithMongoMeta<Payload & { _$hash: Hash; _$meta?: unknown }>[]\n    const result = await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))\n    // console.log(`getHandler: ${JSON.stringify(hashes, null, 2)}:${JSON.stringify(result, null, 2)}`)\n    return result\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<WithMeta<Payload>[]> {\n    const [bw, p] = await validByType(payloads)\n    const payloadsWithExternalMeta = await Promise.all(p.map((value, index) => toDbRepresentation(value, index)))\n    if (payloadsWithExternalMeta.length > 0) {\n      const payloadsResult = await this.payloads.insertMany(payloadsWithExternalMeta)\n      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length)\n        throw new Error('MongoDBArchivist: Error inserting Payloads')\n    }\n    const boundWitnessesWithExternalMeta = await Promise.all(bw.map((value, index) => toDbRepresentation(value, index)))\n    if (boundWitnessesWithExternalMeta.length > 0) {\n      const boundWitnessesResult = await this.boundWitnesses.insertMany(boundWitnessesWithExternalMeta)\n      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length)\n        throw new Error('MongoDBArchivist: Error inserting BoundWitnesses')\n    }\n\n    return await PayloadBuilder.build([...boundWitnessesWithExternalMeta, ...payloadsWithExternalMeta].map(fromDbRepresentation))\n  }\n\n  protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithMeta<Payload>[]> {\n    // Sanitize inputs and set defaults\n    let {\n      limit, offset, order,\n    } = options ?? { limit: 10, order: 'desc' }\n\n    if (!limit) limit = 10\n    if (limit > 100) limit = 100\n\n    if (order != 'asc') order = 'desc'\n\n    let id: ObjectId | undefined\n    if (offset) {\n      const payload = await this.findOneByHash(offset)\n      // TODO: Should we throw an error if the requested payload is not found?\n      if (payload) id = payload._id\n    } else {\n      id = order === 'asc'\n      // If ascending, start from the beginning of time\n        ? ObjectId.createFromTime(0)\n        // If descending, start from now (plus a bit more in the future to ensure\n        // them most recent ObjectIds are included)\n        : ObjectId.createFromTime((Date.now() + 10_000) / 1000)\n    }\n    if (!id) return []\n\n    // Create aggregate criteria\n    const sort = order === 'asc' ? 1 : -1\n    // TODO: How to handle random component of ID across multiple collections\n    // to ensure we don't skip some payloads\n    const match = order === 'asc' ? { _id: { $gt: id } } : { _id: { $lt: id } }\n\n    // Run the aggregate query\n    const foundPayloads = await this.payloads.useCollection((collection) => {\n      return collection\n        .aggregate<PayloadWithMongoMeta>([\n          // Pre-filter payloads collection\n          { $match: match },\n          // Sort payloads by _id\n          { $sort: { _id: sort } },\n          // Limit payloads to the first N payloads\n          { $limit: limit },\n          // Combine with filtered boundWitnesses collection\n          {\n            $unionWith: {\n              coll: this.boundWitnessSdkConfig.collection,\n              pipeline: [\n                { $match: match }, // Pre-filter boundWitnesses\n                { $sort: { _id: sort } }, // Sort boundWitnesses by _id\n                { $limit: limit }, // Limit boundWitnesses to the first N boundWitnesses\n              ],\n            },\n          },\n          // Sort the combined result by _id\n          { $sort: { _id: sort } },\n          // Limit the final result to N documents\n          { $limit: limit },\n        ])\n        .maxTimeMS(this.aggregateTimeoutMs)\n        .toArray()\n    })\n\n    // Convert from DB representation to Payloads\n    return await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    await this.ensureIndexes()\n    return true\n  }\n}\n","import type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness, isQueryBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessWrapper, QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-wrapper'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\n\nexport const validByType = async (payloads: Payload[] = []) => {\n  const results: [BoundWitness[], Payload[]] = [[], []]\n  await Promise.all(\n    payloads.map(async (payload) => {\n      if (isBoundWitness(payload)) {\n        const wrapper = isQueryBoundWitness(payload) ? QueryBoundWitnessWrapper : BoundWitnessWrapper\n        const bw = wrapper.parse(payload)\n        if (await bw.getValid()) {\n          results[0].push(payload)\n        } else {\n          const errors = await bw.getErrors()\n          console.log(`validByType.Error: ${JSON.stringify(errors, null, 2)}`)\n        }\n      } else {\n        const payloadWrapper = PayloadWrapper.wrap(payload)\n        if (await payloadWrapper.getValid()) {\n          results[1].push(payload)\n        }\n      }\n      return\n    }),\n  )\n  return results\n}\n","export * from './Archivist.js'\nexport * from '@xyo-network/archivist-model-mongodb'\n"],"mappings":";;;;AAAA,SAASA,cAAc;AAEvB,SAASC,yBAAyB;AAElC,SAASC,4BAA4BC,gCAAgC;AACrE,SAASC,oCAAoC;AAC7C,SAASC,0BAA0B;AACnC,SAASC,sBAAsB;AAK/B,SAASC,sBAAsBC,0BAA0B;AACzD,SAASC,kBAAAA,uBAAsB;AAC/B,SAASC,gBAAgB;;;ACbzB,SAASC,gBAAgBC,2BAA2B;AACpD,SAASC,qBAAqBC,gCAAgC;AAE9D,SAASC,sBAAsB;AAExB,IAAMC,cAAc,8BAAOC,WAAsB,CAAA,MAAE;AACxD,QAAMC,UAAuC;IAAC,CAAA;IAAI,CAAA;;AAClD,QAAMC,QAAQC,IACZH,SAASI,IAAI,OAAOC,YAAAA;AAClB,QAAIC,eAAeD,OAAAA,GAAU;AAC3B,YAAME,UAAUC,oBAAoBH,OAAAA,IAAWI,2BAA2BC;AAC1E,YAAMC,KAAKJ,QAAQK,MAAMP,OAAAA;AACzB,UAAI,MAAMM,GAAGE,SAAQ,GAAI;AACvBZ,gBAAQ,CAAA,EAAGa,KAAKT,OAAAA;MAClB,OAAO;AACL,cAAMU,SAAS,MAAMJ,GAAGK,UAAS;AACjCC,gBAAQC,IAAI,sBAAsBC,KAAKC,UAAUL,QAAQ,MAAM,CAAA,CAAA,EAAI;MACrE;IACF,OAAO;AACL,YAAMM,iBAAiBC,eAAeC,KAAKlB,OAAAA;AAC3C,UAAI,MAAMgB,eAAeR,SAAQ,GAAI;AACnCZ,gBAAQ,CAAA,EAAGa,KAAKT,OAAAA;MAClB;IACF;AACA;EACF,CAAA,CAAA;AAEF,SAAOJ;AACT,GAvB2B;;;ADY3B,IAAMuB,uBAAuBC,mBAAmBC,iBAAAA;AAEzC,IAAMC,mBAAN,cAA+BH,qBAAAA;EApBtC,OAoBsCA;;;EACpC,OAAyBI,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAErCE,UAAoB;IAACC;IAA4BC;OAA6B,MAAMF;;;;;EAKnFG,qBAAqB;EAExC,MAAeC,OAAqC;AAClD,UAAMA,OAAO,MAAM,KAAKC,KAAK;MAAEC,OAAO;MAAGC,OAAO;IAAO,CAAA;AACvD,WAAOH,KAAK,CAAA,IAAKI,gBAAeC,KAAKL,KAAK,CAAA,CAAE,EAAEM,UAAUC;EAC1D;EAEA,MAAgBC,cAAcC,MAAY;AACxC,UAAMC,cAAe,MAAM,KAAKC,SAASC,QAAQ;MAAEC,QAAQJ;IAAK,CAAA;AAChE,QAAIC,aAAa;AACf,aAAOA;IACT,OAAO;AACL,YAAMI,SAAU,MAAM,KAAKC,eAAeH,QAAQ;QAAEC,QAAQJ;MAAK,CAAA;AACjE,UAAIK,QAAQ;AACV,eAAOA;MACT,OAAO;AACL,cAAMR,UAAW,MAAM,KAAKK,SAASC,QAAQ;UAAEI,OAAOP;QAAK,CAAA;AAC3D,YAAIH,SAAS;AACX,iBAAOA;QACT,OAAO;AACL,gBAAMW,KAAM,MAAM,KAAKF,eAAeH,QAAQ;YAAEI,OAAOP;UAAK,CAAA;AAC5D,iBAAOQ,MAAMV;QACf;MACF;IACF;EACF;EAEA,MAAyBW,WAAWC,QAA8C;AAChF,QAAIC,kBAAkB;SAAID;;AAE1B,UAAME,gBAAgB,MAAMC,QAAQC,IAAIH,gBAAgBI,IAAIX,CAAAA,WAAU,KAAKF,SAASC,QAAQ;MAAEC;IAAO,CAAA,CAAA,CAAA,GAAMY,OAAOC,MAAAA;AAClH,UAAMC,qBAAqB,IAAIC,IAAIP,aAAaG,IAAIlB,CAAAA,YAAWA,QAAQO,MAAM,CAAA;AAC7EO,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACkB,mBAAmBE,IAAIpB,IAAAA,CAAAA;AAEzE,UAAMqB,WAAW,MAAMR,QAAQC,IAAIH,gBAAgBI,IAAIX,CAAAA,WAAU,KAAKE,eAAeH,QAAQ;MAAEC;IAAO,CAAA,CAAA,CAAA,GAAMY,OAAOC,MAAAA;AACnH,UAAMK,gBAAgB,IAAIH,IAAIE,QAAQN,IAAIlB,CAAAA,YAAWA,QAAQO,MAAM,CAAA;AACnEO,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACsB,cAAcF,IAAIpB,IAAAA,CAAAA;AAEpE,UAAME,YAAY,MAAMW,QAAQC,IAAIH,gBAAgBI,IAAIR,CAAAA,UAAS,KAAKL,SAASC,QAAQ;MAAEI;IAAM,CAAA,CAAA,CAAA,GAAMS,OAAOC,MAAAA;AAC5G,UAAMM,iBAAiB,IAAIJ,IAAIjB,SAASa,IAAIlB,CAAAA,YAAWA,QAAQU,KAAK,CAAA;AACpEI,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACuB,eAAeH,IAAIpB,IAAAA,CAAAA;AAErE,UAAMwB,OAAO,MAAMX,QAAQC,IAAIH,gBAAgBI,IAAIR,CAAAA,UAAS,KAAKD,eAAeH,QAAQ;MAAEI;IAAM,CAAA,CAAA,CAAA,GAAMS,OAAOC,MAAAA;AAC7G,UAAMQ,YAAY,IAAIN,IAAIK,IAAIT,IAAIlB,CAAAA,YAAWA,QAAQU,KAAK,CAAA;AAC1DI,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACyB,UAAUL,IAAIpB,IAAAA,CAAAA;AAEhE,UAAM0B,gBAAgB;SAAId;SAAiBS;SAAYnB;SAAasB;;AACpE,UAAMG,SAAS,MAAMC,eAAeC,MAAMH,cAAcX,IAAIe,oBAAAA,CAAAA;AAE5D,WAAOH;EACT;EAEA,MAAyBI,cAAc7B,UAAmD;AACxF,UAAM,CAACM,IAAIwB,CAAAA,IAAK,MAAMC,YAAY/B,QAAAA;AAClC,UAAMgC,2BAA2B,MAAMrB,QAAQC,IAAIkB,EAAEjB,IAAI,CAACoB,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AACrG,QAAIF,yBAAyBI,SAAS,GAAG;AACvC,YAAMC,iBAAiB,MAAM,KAAKrC,SAASsC,WAAWN,wBAAAA;AACtD,UAAI,CAACK,eAAeE,gBAAgBF,eAAeG,kBAAkBR,yBAAyBI,OAC5F,OAAM,IAAIK,MAAM,4CAAA;IACpB;AACA,UAAMC,iCAAiC,MAAM/B,QAAQC,IAAIN,GAAGO,IAAI,CAACoB,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AAC5G,QAAIQ,+BAA+BN,SAAS,GAAG;AAC7C,YAAMO,uBAAuB,MAAM,KAAKvC,eAAekC,WAAWI,8BAAAA;AAClE,UAAI,CAACC,qBAAqBJ,gBAAgBI,qBAAqBH,kBAAkBE,+BAA+BN,OAC9G,OAAM,IAAIK,MAAM,kDAAA;IACpB;AAEA,WAAO,MAAMf,eAAeC,MAAM;SAAIe;SAAmCV;MAA0BnB,IAAIe,oBAAAA,CAAAA;EACzG;EAEA,MAAyBgB,YAAYC,SAA8D;AAEjG,QAAI,EACFtD,OAAOuD,QAAQtD,MAAK,IAClBqD,WAAW;MAAEtD,OAAO;MAAIC,OAAO;IAAO;AAE1C,QAAI,CAACD,MAAOA,SAAQ;AACpB,QAAIA,QAAQ,IAAKA,SAAQ;AAEzB,QAAIC,SAAS,MAAOA,SAAQ;AAE5B,QAAIuD;AACJ,QAAID,QAAQ;AACV,YAAMnD,UAAU,MAAM,KAAKE,cAAciD,MAAAA;AAEzC,UAAInD,QAASoD,MAAKpD,QAAQqD;IAC5B,OAAO;AACLD,WAAKvD,UAAU,QAEXyD,SAASC,eAAe,CAAA,IAGxBD,SAASC,gBAAgBC,KAAKC,IAAG,IAAK,OAAU,GAAA;IACtD;AACA,QAAI,CAACL,GAAI,QAAO,CAAA;AAGhB,UAAMM,OAAO7D,UAAU,QAAQ,IAAI;AAGnC,UAAM8D,QAAQ9D,UAAU,QAAQ;MAAEwD,KAAK;QAAEO,KAAKR;MAAG;IAAE,IAAI;MAAEC,KAAK;QAAEQ,KAAKT;MAAG;IAAE;AAG1E,UAAMvB,gBAAgB,MAAM,KAAKxB,SAASyD,cAAc,CAACC,eAAAA;AACvD,aAAOA,WACJC,UAAgC;;QAE/B;UAAEC,QAAQN;QAAM;;QAEhB;UAAEO,OAAO;YAAEb,KAAKK;UAAK;QAAE;;QAEvB;UAAES,QAAQvE;QAAM;;QAEhB;UACEwE,YAAY;YACVC,MAAM,KAAKC,sBAAsBP;YACjCQ,UAAU;cACR;gBAAEN,QAAQN;cAAM;cAChB;gBAAEO,OAAO;kBAAEb,KAAKK;gBAAK;cAAE;cACvB;gBAAES,QAAQvE;cAAM;;UAEpB;QACF;;QAEA;UAAEsE,OAAO;YAAEb,KAAKK;UAAK;QAAE;;QAEvB;UAAES,QAAQvE;QAAM;OACjB,EACA4E,UAAU,KAAK/E,kBAAkB,EACjCgF,QAAO;IACZ,CAAA;AAGA,WAAO,MAAM1C,eAAeC,MAAMH,cAAcX,IAAIe,oBAAAA,CAAAA;EACtD;EAEA,MAAyByC,eAAe;AACtC,UAAM,MAAMA,aAAAA;AACZ,UAAM,KAAKC,cAAa;AACxB,WAAO;EACT;AACF;;;AEzKA,cAAc;","names":["exists","AbstractArchivist","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","MongoDBArchivistConfigSchema","MongoDBModuleMixin","PayloadBuilder","fromDbRepresentation","toDbRepresentation","PayloadWrapper","ObjectId","isBoundWitness","isQueryBoundWitness","BoundWitnessWrapper","QueryBoundWitnessWrapper","PayloadWrapper","validByType","payloads","results","Promise","all","map","payload","isBoundWitness","wrapper","isQueryBoundWitness","QueryBoundWitnessWrapper","BoundWitnessWrapper","bw","parse","getValid","push","errors","getErrors","console","log","JSON","stringify","payloadWrapper","PayloadWrapper","wrap","MongoDBArchivistBase","MongoDBModuleMixin","AbstractArchivist","MongoDBArchivist","configSchemas","MongoDBArchivistConfigSchema","defaultConfigSchema","queries","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","aggregateTimeoutMs","head","next","limit","order","PayloadWrapper","wrap","payload","undefined","findOneByHash","hash","dataPayload","payloads","findOne","_$hash","dataBw","boundWitnesses","_hash","bw","getHandler","hashes","remainingHashes","dataPayloads","Promise","all","map","filter","exists","dataPayloadsHashes","Set","has","dataBws","dataBwsHashes","payloadsHashes","bws","bwsHashes","foundPayloads","result","PayloadBuilder","build","fromDbRepresentation","insertHandler","p","validByType","payloadsWithExternalMeta","value","index","toDbRepresentation","length","payloadsResult","insertMany","acknowledged","insertedCount","Error","boundWitnessesWithExternalMeta","boundWitnessesResult","nextHandler","options","offset","id","_id","ObjectId","createFromTime","Date","now","sort","match","$gt","$lt","useCollection","collection","aggregate","$match","$sort","$limit","$unionWith","coll","boundWitnessSdkConfig","pipeline","maxTimeMS","toArray","startHandler","ensureIndexes"]}

package/dist/node/Archivist.d.ts

@@ -1,5 +1,6 @@
 import type { Hash } from '@xylabs/hex';
 import { AbstractArchivist } from '@xyo-network/archivist-abstract';
+import type { ArchivistNextOptions } from '@xyo-network/archivist-model';
 import type { Payload, Schema, WithMeta } from '@xyo-network/payload-model';
 import type { PayloadWithMongoMeta } from '@xyo-network/payload-mongodb';
 declare const MongoDBArchivistBase: (abstract new (...args: any[]) => {
@@ -67,9 +68,15 @@ export declare class MongoDBArchivist extends MongoDBArchivistBase {
     static readonly configSchemas: Schema[];
     static readonly defaultConfigSchema: Schema;
     readonly queries: string[];
+    /**
+     * The amount of time to allow the aggregate query to execute
+     */
+    protected readonly aggregateTimeoutMs = 10000;
     head(): Promise<Payload | undefined>;
+    protected findOneByHash(hash: Hash): Promise<import("mongodb").WithId<PayloadWithMongoMeta> | undefined>;
     protected getHandler(hashes: Hash[]): Promise<WithMeta<Payload>[]>;
     protected insertHandler(payloads: Payload[]): Promise<WithMeta<Payload>[]>;
+    protected nextHandler(options?: ArchivistNextOptions): Promise<WithMeta<Payload>[]>;
     protected startHandler(): Promise<boolean>;
 }
 export {};

package/dist/node/Archivist.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Archivist.d.ts","sourceRoot":"","sources":["../../src/Archivist.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AAKnE,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAAE,QAAQ,EAC1B,MAAM,4BAA4B,CAAA;AACnC,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAA;AAMxE,QAAA,MAAM,oBAAoB;;;;;8BAgCoD,8BAA8B;;;;oBAOtE,aACnC;;+CA/B0E,+BACnE;0CAGQ,+BAA+B;uBACtB,mCAEJ;;;;;;;qBAwB2D,2BAA2B;;yBA7CrF,4BAA4B;;mCACG,4BAEnC;;uBAE6B,4BAExC,wBAAwB,4BACjC,8BAA8B,2BAA2B,iCAAiC,2BACjF,8BAA8B,4BAA4B,wBAE/D,4BAA4B,8BAA8B,2BAE7D,iCAAiC,2BAC7B;iCAAsD,CAAC,iBAAiB,+BACnE;4BAGR,CAAD,iBAAiB,+BAA+B;2BACtB,mCAEJ;;;;;;;8CAE+B,2BAA2B;;wCAEV,+BAA+B,6CAClF,+BAA+B;0BACV,8BAA8B;mCAErC,+BAA+B,6CAA6C,+BACzF;;+BAiBP,iBAAiB;;6BACA,iCAAiC,6BAA6B,iCAC3E,0CAA0C,2BACxD,wBACkB,2BAA2B,kCAAkC,CAAC,SAAS,4BAC7F,wBAAwB,CAAC,mBAAmB,iBACpC,oBAAoB,2BAA2B;iCACjD,iCAAiC,6BAA6B,iCAAiC,4CAClF,2BAA2B,wBAAwB,2BAGrE,kCAAkC,CAAC,SAAS,4BAA4B,wBAAwB,CAAC,qBAGxF,iBAAiB;yBAEvB,iBAAa;wBAIN,iBAAiB;sBAA6C,2BAA2B;4CAAuE,2BAA2B,0CAA0C,2BAA2B;yCAA8E,2BAA2B,oEAAoE,2BAA2B;+CAAsG,2BAA2B,oEAAoE,2BAA2B;2CAAkG,2BAA2B,0CAA0C,2BAA2B;wCAA+E,2BAA2B,mFAAmF,4BAA4B,uBAAuB,2BAA2B;4BAA2E,4BAA4B;uCAAkF,2BAA2B,mFAAmF,4BAA4B,uBAAuB,2BAA2B,gDAAgD,4BAA4B;2BAAuD,4BAA4B,4BAA4B,4BAA4B;yCAAqE,2BAA2B,mEAAmE,4BAA4B,uBAAuB,2BAA2B,gDAAgD,4BAA4B;;;4BA9Dv8D,CAAA;AAElE,qBAAa,gBAAiB,SAAQ,oBAAoB;IACxD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAyD;IACzG,gBAAyB,mBAAmB,EAAE,MAAM,CAA+B;IAEnF,SAAkB,OAAO,EAAE,MAAM,EAAE,CAAiD;IAErE,IAAI,IAAI,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC;cAK1B,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAyBxD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAkBhE,YAAY;CAKtC"}
+{"version":3,"file":"Archivist.d.ts","sourceRoot":"","sources":["../../src/Archivist.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAA;AAKxE,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAAE,QAAQ,EAC1B,MAAM,4BAA4B,CAAA;AACnC,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAA;AAOxE,QAAA,MAAM,oBAAoB;;;;;8BA6CsC,8BAA8B;;;;oBAMzF,aAAa;;+CA7CqE,+BAEnF;0CAGQ,+BAA8B;uBAGtC,mCAA+B;;;;;;;qBAuCnB,2BAA2B;;yBA7DT,4BACR;;mCACkB,4BAA4B;;uBACb,4BAClC,wBAEZ,4BAA4B,8BAEhB,2BAA2B,iCAEnC,2BAA2B,8BAC/B,4BAA4B,wBAAwB,4BAA4B,8BAC9E,2BAA2B,iCAAiC,2BAE5D;iCAAsD,CAAC,iBAAiB,+BAEnF;4BAED,CAAH,iBACY,+BAA8B;2BAGtC,mCAA+B;;;;;;;8CAMd,2BAErB;;wCAC8D,+BAC5C,6CAGV,+BAA6B;0BAE3B,8BACE;mCAC0B,+BAA+B,6CAE7D,+BAKC;;+BAeoG,iBACxG;;6BACqB,iCAAiC,6BAE/C,iCAAiC,0CAA0C,2BAA2B,wBAAwB,2BAC1H,kCAAkC,CAAC,SAAS,4BAC3D,wBAAwB,CAAC,mBAAmB,iBAAiB,oBAAoB,2BAC7E;iCAGgC,iCAAiC,6BACjE,iCAAiC,4CACH,2BAA0B,wBAAwB,2BAA2B,kCAChF,CAAC,SAAS,4BACnB,wBAAwB,CAAC,qBAAqB,iBACrE;yBAAgD,iBAAiB;wBACzD,iBAAa;sBAA6C,2BAEjD;4CAAuE,2BAA2B,0CAC/E,2BAC1B;yCAA8E,2BAC7E,oEAAoE,2BAA2B;+CAItG,2BAAsB,oEAAoE,2BAA2B;2CAGrC,2BAC5E,0CAEF,2BAAqB;wCAIvB,2BAAyB,mFAKxB,4BACU,uBAAuB,2BAC5B;4BACJ,4BAA4B;uCAGM,2BAC7B,mFAC+C,4BACvD,uBAAuB,2BACzB,gDAAgD,4BAE1C;2BAGF,4BAA4B,4BACb,4BAA2B;yCACT,2BAClB,mEAEb,4BACJ,uBAAuB,2BAA2B,gDAE7C,4BAAmB;;;4BApHuC,CAAA;AAElE,qBAAa,gBAAiB,SAAQ,oBAAoB;IACxD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAyD;IACzG,gBAAyB,mBAAmB,EAAE,MAAM,CAA+B;IAEnF,SAAkB,OAAO,EAAE,MAAM,EAAE,CAA2E;IAE9G;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,kBAAkB,SAAS;IAE/B,IAAI,IAAI,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC;cAKnC,aAAa,CAAC,IAAI,EAAE,IAAI;cAoBf,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAyBxD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAkBhE,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;cAkEzE,YAAY;CAKtC"}

package/dist/node/index.mjs

@@ -4,12 +4,13 @@ var __name = (target, value) => __defProp(target, "name", { value, configurable:
 // src/Archivist.ts
 import { exists } from "@xylabs/exists";
 import { AbstractArchivist } from "@xyo-network/archivist-abstract";
-import { ArchivistInsertQuerySchema } from "@xyo-network/archivist-model";
+import { ArchivistInsertQuerySchema, ArchivistNextQuerySchema } from "@xyo-network/archivist-model";
 import { MongoDBArchivistConfigSchema } from "@xyo-network/archivist-model-mongodb";
 import { MongoDBModuleMixin } from "@xyo-network/module-abstract-mongodb";
 import { PayloadBuilder } from "@xyo-network/payload-builder";
 import { fromDbRepresentation, toDbRepresentation } from "@xyo-network/payload-mongodb";
 import { PayloadWrapper as PayloadWrapper2 } from "@xyo-network/payload-wrapper";
+import { ObjectId } from "mongodb";
 
 // src/lib/validByType.ts
 import { isBoundWitness, isQueryBoundWitness } from "@xyo-network/boundwitness-model";
@@ -25,7 +26,7 @@ var validByType = /* @__PURE__ */ __name(async (payloads = []) => {
       const wrapper = isQueryBoundWitness(payload) ? QueryBoundWitnessWrapper : BoundWitnessWrapper;
       const bw = wrapper.parse(payload);
       if (await bw.getValid()) {
-        results[0].push(bw.payload);
+        results[0].push(payload);
       } else {
         const errors = await bw.getErrors();
         console.log(`validByType.Error: ${JSON.stringify(errors, null, 2)}`);
@@ -33,7 +34,7 @@ var validByType = /* @__PURE__ */ __name(async (payloads = []) => {
     } else {
       const payloadWrapper = PayloadWrapper.wrap(payload);
       if (await payloadWrapper.getValid()) {
-        results[1].push(payloadWrapper.payload);
+        results[1].push(payload);
       }
     }
     return;
@@ -54,14 +55,47 @@ var MongoDBArchivist = class extends MongoDBArchivistBase {
   static defaultConfigSchema = MongoDBArchivistConfigSchema;
   queries = [
     ArchivistInsertQuerySchema,
+    ArchivistNextQuerySchema,
     ...super.queries
   ];
+  /**
+  * The amount of time to allow the aggregate query to execute
+  */
+  aggregateTimeoutMs = 1e4;
   async head() {
-    const head = await (await this.payloads.find({})).sort({
-      _timestamp: -1
-    }).limit(1).toArray();
+    const head = await this.next({
+      limit: 1,
+      order: "desc"
+    });
     return head[0] ? PayloadWrapper2.wrap(head[0]).payload : void 0;
   }
+  async findOneByHash(hash) {
+    const dataPayload = await this.payloads.findOne({
+      _$hash: hash
+    });
+    if (dataPayload) {
+      return dataPayload;
+    } else {
+      const dataBw = await this.boundWitnesses.findOne({
+        _$hash: hash
+      });
+      if (dataBw) {
+        return dataBw;
+      } else {
+        const payload = await this.payloads.findOne({
+          _hash: hash
+        });
+        if (payload) {
+          return payload;
+        } else {
+          const bw = await this.boundWitnesses.findOne({
+            _hash: hash
+          });
+          return bw ?? void 0;
+        }
+      }
+    }
+  }
   async getHandler(hashes) {
     let remainingHashes = [
       ...hashes
@@ -100,17 +134,92 @@ var MongoDBArchivist = class extends MongoDBArchivistBase {
     const payloadsWithExternalMeta = await Promise.all(p.map((value, index) => toDbRepresentation(value, index)));
     if (payloadsWithExternalMeta.length > 0) {
       const payloadsResult = await this.payloads.insertMany(payloadsWithExternalMeta);
-      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length) throw new Error("MongoDBDeterministicArchivist: Error inserting Payloads");
+      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length) throw new Error("MongoDBArchivist: Error inserting Payloads");
     }
     const boundWitnessesWithExternalMeta = await Promise.all(bw.map((value, index) => toDbRepresentation(value, index)));
     if (boundWitnessesWithExternalMeta.length > 0) {
       const boundWitnessesResult = await this.boundWitnesses.insertMany(boundWitnessesWithExternalMeta);
-      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length) throw new Error("MongoDBDeterministicArchivist: Error inserting BoundWitnesses");
+      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length) throw new Error("MongoDBArchivist: Error inserting BoundWitnesses");
     }
     return await PayloadBuilder.build([
       ...boundWitnessesWithExternalMeta,
       ...payloadsWithExternalMeta
-    ]);
+    ].map(fromDbRepresentation));
+  }
+  async nextHandler(options) {
+    let { limit, offset, order } = options ?? {
+      limit: 10,
+      order: "desc"
+    };
+    if (!limit) limit = 10;
+    if (limit > 100) limit = 100;
+    if (order != "asc") order = "desc";
+    let id;
+    if (offset) {
+      const payload = await this.findOneByHash(offset);
+      if (payload) id = payload._id;
+    } else {
+      id = order === "asc" ? ObjectId.createFromTime(0) : ObjectId.createFromTime((Date.now() + 1e4) / 1e3);
+    }
+    if (!id) return [];
+    const sort = order === "asc" ? 1 : -1;
+    const match = order === "asc" ? {
+      _id: {
+        $gt: id
+      }
+    } : {
+      _id: {
+        $lt: id
+      }
+    };
+    const foundPayloads = await this.payloads.useCollection((collection) => {
+      return collection.aggregate([
+        // Pre-filter payloads collection
+        {
+          $match: match
+        },
+        // Sort payloads by _id
+        {
+          $sort: {
+            _id: sort
+          }
+        },
+        // Limit payloads to the first N payloads
+        {
+          $limit: limit
+        },
+        // Combine with filtered boundWitnesses collection
+        {
+          $unionWith: {
+            coll: this.boundWitnessSdkConfig.collection,
+            pipeline: [
+              {
+                $match: match
+              },
+              {
+                $sort: {
+                  _id: sort
+                }
+              },
+              {
+                $limit: limit
+              }
+            ]
+          }
+        },
+        // Sort the combined result by _id
+        {
+          $sort: {
+            _id: sort
+          }
+        },
+        // Limit the final result to N documents
+        {
+          $limit: limit
+        }
+      ]).maxTimeMS(this.aggregateTimeoutMs).toArray();
+    });
+    return await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation));
   }
   async startHandler() {
     await super.startHandler();

package/dist/node/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Archivist.ts","../../src/lib/validByType.ts","../../src/index.ts"],"sourcesContent":["import { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport { ArchivistInsertQuerySchema } from '@xyo-network/archivist-model'\nimport { MongoDBArchivistConfigSchema } from '@xyo-network/archivist-model-mongodb'\nimport { MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema, WithMeta,\n} from '@xyo-network/payload-model'\nimport type { PayloadWithMongoMeta } from '@xyo-network/payload-mongodb'\nimport { fromDbRepresentation, toDbRepresentation } from '@xyo-network/payload-mongodb'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\n\nimport { validByType } from './lib/index.js'\n\nconst MongoDBArchivistBase = MongoDBModuleMixin(AbstractArchivist)\n\nexport class MongoDBArchivist extends MongoDBArchivistBase {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, MongoDBArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = MongoDBArchivistConfigSchema\n\n  override readonly queries: string[] = [ArchivistInsertQuerySchema, ...super.queries]\n\n  override async head(): Promise<Payload | undefined> {\n    const head = await (await this.payloads.find({})).sort({ _timestamp: -1 }).limit(1).toArray()\n    return head[0] ? PayloadWrapper.wrap(head[0]).payload : undefined\n  }\n\n  protected override async getHandler(hashes: Hash[]): Promise<WithMeta<Payload>[]> {\n    let remainingHashes = [...hashes]\n\n    const dataPayloads = (await Promise.all(remainingHashes.map(_$hash => this.payloads.findOne({ _$hash })))).filter(exists)\n    const dataPayloadsHashes = new Set(dataPayloads.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataPayloadsHashes.has(hash))\n\n    const dataBws = (await Promise.all(remainingHashes.map(_$hash => this.boundWitnesses.findOne({ _$hash })))).filter(exists)\n    const dataBwsHashes = new Set(dataBws.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataBwsHashes.has(hash))\n\n    const payloads = (await Promise.all(remainingHashes.map(_hash => this.payloads.findOne({ _hash })))).filter(exists)\n    const payloadsHashes = new Set(payloads.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !payloadsHashes.has(hash))\n\n    const bws = (await Promise.all(remainingHashes.map(_hash => this.boundWitnesses.findOne({ _hash })))).filter(exists)\n    const bwsHashes = new Set(bws.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !bwsHashes.has(hash))\n\n    const foundPayloads = [...dataPayloads, ...dataBws, ...payloads, ...bws] as PayloadWithMongoMeta<Payload & { _$hash: Hash; _$meta?: unknown }>[]\n    const result = await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))\n    // console.log(`getHandler: ${JSON.stringify(hashes, null, 2)}:${JSON.stringify(result, null, 2)}`)\n    return result\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<WithMeta<Payload>[]> {\n    const [bw, p] = await validByType(payloads)\n    const payloadsWithExternalMeta = await Promise.all(p.map((value, index) => toDbRepresentation(value, index)))\n    if (payloadsWithExternalMeta.length > 0) {\n      const payloadsResult = await this.payloads.insertMany(payloadsWithExternalMeta)\n      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length)\n        throw new Error('MongoDBDeterministicArchivist: Error inserting Payloads')\n    }\n    const boundWitnessesWithExternalMeta = await Promise.all(bw.map((value, index) => toDbRepresentation(value, index)))\n    if (boundWitnessesWithExternalMeta.length > 0) {\n      const boundWitnessesResult = await this.boundWitnesses.insertMany(boundWitnessesWithExternalMeta)\n      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length)\n        throw new Error('MongoDBDeterministicArchivist: Error inserting BoundWitnesses')\n    }\n\n    return await PayloadBuilder.build([...boundWitnessesWithExternalMeta, ...payloadsWithExternalMeta])\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    await this.ensureIndexes()\n    return true\n  }\n}\n","import type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness, isQueryBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessWrapper, QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-wrapper'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\n\nexport const validByType = async (payloads: Payload[] = []) => {\n  const results: [BoundWitness[], Payload[]] = [[], []]\n  await Promise.all(\n    payloads.map(async (payload) => {\n      if (isBoundWitness(payload)) {\n        const wrapper = isQueryBoundWitness(payload) ? QueryBoundWitnessWrapper : BoundWitnessWrapper\n        const bw = wrapper.parse(payload)\n        if (await bw.getValid()) {\n          results[0].push(bw.payload)\n        } else {\n          const errors = await bw.getErrors()\n          console.log(`validByType.Error: ${JSON.stringify(errors, null, 2)}`)\n        }\n      } else {\n        const payloadWrapper = PayloadWrapper.wrap(payload)\n        if (await payloadWrapper.getValid()) {\n          results[1].push(payloadWrapper.payload)\n        }\n      }\n      return\n    }),\n  )\n  return results\n}\n","export * from './Archivist.js'\nexport * from '@xyo-network/archivist-model-mongodb'\n"],"mappings":";;;;AAAA,SAASA,cAAc;AAEvB,SAASC,yBAAyB;AAClC,SAASC,kCAAkC;AAC3C,SAASC,oCAAoC;AAC7C,SAASC,0BAA0B;AACnC,SAASC,sBAAsB;AAK/B,SAASC,sBAAsBC,0BAA0B;AACzD,SAASC,kBAAAA,uBAAsB;;;ACX/B,SAASC,gBAAgBC,2BAA2B;AACpD,SAASC,qBAAqBC,gCAAgC;AAE9D,SAASC,sBAAsB;AAExB,IAAMC,cAAc,8BAAOC,WAAsB,CAAA,MAAE;AACxD,QAAMC,UAAuC;IAAC,CAAA;IAAI,CAAA;;AAClD,QAAMC,QAAQC,IACZH,SAASI,IAAI,OAAOC,YAAAA;AAClB,QAAIC,eAAeD,OAAAA,GAAU;AAC3B,YAAME,UAAUC,oBAAoBH,OAAAA,IAAWI,2BAA2BC;AAC1E,YAAMC,KAAKJ,QAAQK,MAAMP,OAAAA;AACzB,UAAI,MAAMM,GAAGE,SAAQ,GAAI;AACvBZ,gBAAQ,CAAA,EAAGa,KAAKH,GAAGN,OAAO;MAC5B,OAAO;AACL,cAAMU,SAAS,MAAMJ,GAAGK,UAAS;AACjCC,gBAAQC,IAAI,sBAAsBC,KAAKC,UAAUL,QAAQ,MAAM,CAAA,CAAA,EAAI;MACrE;IACF,OAAO;AACL,YAAMM,iBAAiBC,eAAeC,KAAKlB,OAAAA;AAC3C,UAAI,MAAMgB,eAAeR,SAAQ,GAAI;AACnCZ,gBAAQ,CAAA,EAAGa,KAAKO,eAAehB,OAAO;MACxC;IACF;AACA;EACF,CAAA,CAAA;AAEF,SAAOJ;AACT,GAvB2B;;;ADU3B,IAAMuB,uBAAuBC,mBAAmBC,iBAAAA;AAEzC,IAAMC,mBAAN,cAA+BH,qBAAAA;EAlBtC,OAkBsCA;;;EACpC,OAAyBI,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAErCE,UAAoB;IAACC;OAA+B,MAAMD;;EAE5E,MAAeE,OAAqC;AAClD,UAAMA,OAAO,OAAO,MAAM,KAAKC,SAASC,KAAK,CAAC,CAAA,GAAIC,KAAK;MAAEC,YAAY;IAAG,CAAA,EAAGC,MAAM,CAAA,EAAGC,QAAO;AAC3F,WAAON,KAAK,CAAA,IAAKO,gBAAeC,KAAKR,KAAK,CAAA,CAAE,EAAES,UAAUC;EAC1D;EAEA,MAAyBC,WAAWC,QAA8C;AAChF,QAAIC,kBAAkB;SAAID;;AAE1B,UAAME,gBAAgB,MAAMC,QAAQC,IAAIH,gBAAgBI,IAAIC,CAAAA,WAAU,KAAKjB,SAASkB,QAAQ;MAAED;IAAO,CAAA,CAAA,CAAA,GAAME,OAAOC,MAAAA;AAClH,UAAMC,qBAAqB,IAAIC,IAAIT,aAAaG,IAAIR,CAAAA,YAAWA,QAAQS,MAAM,CAAA;AAC7EL,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACF,mBAAmBG,IAAID,IAAAA,CAAAA;AAEzE,UAAME,WAAW,MAAMX,QAAQC,IAAIH,gBAAgBI,IAAIC,CAAAA,WAAU,KAAKS,eAAeR,QAAQ;MAAED;IAAO,CAAA,CAAA,CAAA,GAAME,OAAOC,MAAAA;AACnH,UAAMO,gBAAgB,IAAIL,IAAIG,QAAQT,IAAIR,CAAAA,YAAWA,QAAQS,MAAM,CAAA;AACnEL,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACI,cAAcH,IAAID,IAAAA,CAAAA;AAEpE,UAAMvB,YAAY,MAAMc,QAAQC,IAAIH,gBAAgBI,IAAIY,CAAAA,UAAS,KAAK5B,SAASkB,QAAQ;MAAEU;IAAM,CAAA,CAAA,CAAA,GAAMT,OAAOC,MAAAA;AAC5G,UAAMS,iBAAiB,IAAIP,IAAItB,SAASgB,IAAIR,CAAAA,YAAWA,QAAQoB,KAAK,CAAA;AACpEhB,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACM,eAAeL,IAAID,IAAAA,CAAAA;AAErE,UAAMO,OAAO,MAAMhB,QAAQC,IAAIH,gBAAgBI,IAAIY,CAAAA,UAAS,KAAKF,eAAeR,QAAQ;MAAEU;IAAM,CAAA,CAAA,CAAA,GAAMT,OAAOC,MAAAA;AAC7G,UAAMW,YAAY,IAAIT,IAAIQ,IAAId,IAAIR,CAAAA,YAAWA,QAAQoB,KAAK,CAAA;AAC1DhB,sBAAkBA,gBAAgBO,OAAOI,CAAAA,SAAQ,CAACQ,UAAUP,IAAID,IAAAA,CAAAA;AAEhE,UAAMS,gBAAgB;SAAInB;SAAiBY;SAAYzB;SAAa8B;;AACpE,UAAMG,SAAS,MAAMC,eAAeC,MAAMH,cAAchB,IAAIoB,oBAAAA,CAAAA;AAE5D,WAAOH;EACT;EAEA,MAAyBI,cAAcrC,UAAmD;AACxF,UAAM,CAACsC,IAAIC,CAAAA,IAAK,MAAMC,YAAYxC,QAAAA;AAClC,UAAMyC,2BAA2B,MAAM3B,QAAQC,IAAIwB,EAAEvB,IAAI,CAAC0B,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AACrG,QAAIF,yBAAyBI,SAAS,GAAG;AACvC,YAAMC,iBAAiB,MAAM,KAAK9C,SAAS+C,WAAWN,wBAAAA;AACtD,UAAI,CAACK,eAAeE,gBAAgBF,eAAeG,kBAAkBR,yBAAyBI,OAC5F,OAAM,IAAIK,MAAM,yDAAA;IACpB;AACA,UAAMC,iCAAiC,MAAMrC,QAAQC,IAAIuB,GAAGtB,IAAI,CAAC0B,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AAC5G,QAAIQ,+BAA+BN,SAAS,GAAG;AAC7C,YAAMO,uBAAuB,MAAM,KAAK1B,eAAeqB,WAAWI,8BAAAA;AAClE,UAAI,CAACC,qBAAqBJ,gBAAgBI,qBAAqBH,kBAAkBE,+BAA+BN,OAC9G,OAAM,IAAIK,MAAM,+DAAA;IACpB;AAEA,WAAO,MAAMhB,eAAeC,MAAM;SAAIgB;SAAmCV;KAAyB;EACpG;EAEA,MAAyBY,eAAe;AACtC,UAAM,MAAMA,aAAAA;AACZ,UAAM,KAAKC,cAAa;AACxB,WAAO;EACT;AACF;;;AE5EA,cAAc;","names":["exists","AbstractArchivist","ArchivistInsertQuerySchema","MongoDBArchivistConfigSchema","MongoDBModuleMixin","PayloadBuilder","fromDbRepresentation","toDbRepresentation","PayloadWrapper","isBoundWitness","isQueryBoundWitness","BoundWitnessWrapper","QueryBoundWitnessWrapper","PayloadWrapper","validByType","payloads","results","Promise","all","map","payload","isBoundWitness","wrapper","isQueryBoundWitness","QueryBoundWitnessWrapper","BoundWitnessWrapper","bw","parse","getValid","push","errors","getErrors","console","log","JSON","stringify","payloadWrapper","PayloadWrapper","wrap","MongoDBArchivistBase","MongoDBModuleMixin","AbstractArchivist","MongoDBArchivist","configSchemas","MongoDBArchivistConfigSchema","defaultConfigSchema","queries","ArchivistInsertQuerySchema","head","payloads","find","sort","_timestamp","limit","toArray","PayloadWrapper","wrap","payload","undefined","getHandler","hashes","remainingHashes","dataPayloads","Promise","all","map","_$hash","findOne","filter","exists","dataPayloadsHashes","Set","hash","has","dataBws","boundWitnesses","dataBwsHashes","_hash","payloadsHashes","bws","bwsHashes","foundPayloads","result","PayloadBuilder","build","fromDbRepresentation","insertHandler","bw","p","validByType","payloadsWithExternalMeta","value","index","toDbRepresentation","length","payloadsResult","insertMany","acknowledged","insertedCount","Error","boundWitnessesWithExternalMeta","boundWitnessesResult","startHandler","ensureIndexes"]}
+{"version":3,"sources":["../../src/Archivist.ts","../../src/lib/validByType.ts","../../src/index.ts"],"sourcesContent":["import { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport type { ArchivistNextOptions } from '@xyo-network/archivist-model'\nimport { ArchivistInsertQuerySchema, ArchivistNextQuerySchema } from '@xyo-network/archivist-model'\nimport { MongoDBArchivistConfigSchema } from '@xyo-network/archivist-model-mongodb'\nimport { MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n  Payload, Schema, WithMeta,\n} from '@xyo-network/payload-model'\nimport type { PayloadWithMongoMeta } from '@xyo-network/payload-mongodb'\nimport { fromDbRepresentation, toDbRepresentation } from '@xyo-network/payload-mongodb'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\nimport { ObjectId } from 'mongodb'\n\nimport { validByType } from './lib/index.js'\n\nconst MongoDBArchivistBase = MongoDBModuleMixin(AbstractArchivist)\n\nexport class MongoDBArchivist extends MongoDBArchivistBase {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, MongoDBArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = MongoDBArchivistConfigSchema\n\n  override readonly queries: string[] = [ArchivistInsertQuerySchema, ArchivistNextQuerySchema, ...super.queries]\n\n  /**\n   * The amount of time to allow the aggregate query to execute\n   */\n  protected readonly aggregateTimeoutMs = 10_000\n\n  override async head(): Promise<Payload | undefined> {\n    const head = await this.next({ limit: 1, order: 'desc' })\n    return head[0] ? PayloadWrapper.wrap(head[0]).payload : undefined\n  }\n\n  protected async findOneByHash(hash: Hash) {\n    const dataPayload = (await this.payloads.findOne({ _$hash: hash }))\n    if (dataPayload) {\n      return dataPayload\n    } else {\n      const dataBw = (await this.boundWitnesses.findOne({ _$hash: hash }))\n      if (dataBw) {\n        return dataBw\n      } else {\n        const payload = (await this.payloads.findOne({ _hash: hash }))\n        if (payload) {\n          return payload\n        } else {\n          const bw = (await this.boundWitnesses.findOne({ _hash: hash }))\n          return bw ?? undefined\n        }\n      }\n    }\n  }\n\n  protected override async getHandler(hashes: Hash[]): Promise<WithMeta<Payload>[]> {\n    let remainingHashes = [...hashes]\n\n    const dataPayloads = (await Promise.all(remainingHashes.map(_$hash => this.payloads.findOne({ _$hash })))).filter(exists)\n    const dataPayloadsHashes = new Set(dataPayloads.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataPayloadsHashes.has(hash))\n\n    const dataBws = (await Promise.all(remainingHashes.map(_$hash => this.boundWitnesses.findOne({ _$hash })))).filter(exists)\n    const dataBwsHashes = new Set(dataBws.map(payload => payload._$hash))\n    remainingHashes = remainingHashes.filter(hash => !dataBwsHashes.has(hash))\n\n    const payloads = (await Promise.all(remainingHashes.map(_hash => this.payloads.findOne({ _hash })))).filter(exists)\n    const payloadsHashes = new Set(payloads.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !payloadsHashes.has(hash))\n\n    const bws = (await Promise.all(remainingHashes.map(_hash => this.boundWitnesses.findOne({ _hash })))).filter(exists)\n    const bwsHashes = new Set(bws.map(payload => payload._hash))\n    remainingHashes = remainingHashes.filter(hash => !bwsHashes.has(hash))\n\n    const foundPayloads = [...dataPayloads, ...dataBws, ...payloads, ...bws] as PayloadWithMongoMeta<Payload & { _$hash: Hash; _$meta?: unknown }>[]\n    const result = await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))\n    // console.log(`getHandler: ${JSON.stringify(hashes, null, 2)}:${JSON.stringify(result, null, 2)}`)\n    return result\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<WithMeta<Payload>[]> {\n    const [bw, p] = await validByType(payloads)\n    const payloadsWithExternalMeta = await Promise.all(p.map((value, index) => toDbRepresentation(value, index)))\n    if (payloadsWithExternalMeta.length > 0) {\n      const payloadsResult = await this.payloads.insertMany(payloadsWithExternalMeta)\n      if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length)\n        throw new Error('MongoDBArchivist: Error inserting Payloads')\n    }\n    const boundWitnessesWithExternalMeta = await Promise.all(bw.map((value, index) => toDbRepresentation(value, index)))\n    if (boundWitnessesWithExternalMeta.length > 0) {\n      const boundWitnessesResult = await this.boundWitnesses.insertMany(boundWitnessesWithExternalMeta)\n      if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length)\n        throw new Error('MongoDBArchivist: Error inserting BoundWitnesses')\n    }\n\n    return await PayloadBuilder.build([...boundWitnessesWithExternalMeta, ...payloadsWithExternalMeta].map(fromDbRepresentation))\n  }\n\n  protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithMeta<Payload>[]> {\n    // Sanitize inputs and set defaults\n    let {\n      limit, offset, order,\n    } = options ?? { limit: 10, order: 'desc' }\n\n    if (!limit) limit = 10\n    if (limit > 100) limit = 100\n\n    if (order != 'asc') order = 'desc'\n\n    let id: ObjectId | undefined\n    if (offset) {\n      const payload = await this.findOneByHash(offset)\n      // TODO: Should we throw an error if the requested payload is not found?\n      if (payload) id = payload._id\n    } else {\n      id = order === 'asc'\n      // If ascending, start from the beginning of time\n        ? ObjectId.createFromTime(0)\n        // If descending, start from now (plus a bit more in the future to ensure\n        // them most recent ObjectIds are included)\n        : ObjectId.createFromTime((Date.now() + 10_000) / 1000)\n    }\n    if (!id) return []\n\n    // Create aggregate criteria\n    const sort = order === 'asc' ? 1 : -1\n    // TODO: How to handle random component of ID across multiple collections\n    // to ensure we don't skip some payloads\n    const match = order === 'asc' ? { _id: { $gt: id } } : { _id: { $lt: id } }\n\n    // Run the aggregate query\n    const foundPayloads = await this.payloads.useCollection((collection) => {\n      return collection\n        .aggregate<PayloadWithMongoMeta>([\n          // Pre-filter payloads collection\n          { $match: match },\n          // Sort payloads by _id\n          { $sort: { _id: sort } },\n          // Limit payloads to the first N payloads\n          { $limit: limit },\n          // Combine with filtered boundWitnesses collection\n          {\n            $unionWith: {\n              coll: this.boundWitnessSdkConfig.collection,\n              pipeline: [\n                { $match: match }, // Pre-filter boundWitnesses\n                { $sort: { _id: sort } }, // Sort boundWitnesses by _id\n                { $limit: limit }, // Limit boundWitnesses to the first N boundWitnesses\n              ],\n            },\n          },\n          // Sort the combined result by _id\n          { $sort: { _id: sort } },\n          // Limit the final result to N documents\n          { $limit: limit },\n        ])\n        .maxTimeMS(this.aggregateTimeoutMs)\n        .toArray()\n    })\n\n    // Convert from DB representation to Payloads\n    return await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    await this.ensureIndexes()\n    return true\n  }\n}\n","import type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { isBoundWitness, isQueryBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessWrapper, QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-wrapper'\nimport type { Payload } from '@xyo-network/payload-model'\nimport { PayloadWrapper } from '@xyo-network/payload-wrapper'\n\nexport const validByType = async (payloads: Payload[] = []) => {\n  const results: [BoundWitness[], Payload[]] = [[], []]\n  await Promise.all(\n    payloads.map(async (payload) => {\n      if (isBoundWitness(payload)) {\n        const wrapper = isQueryBoundWitness(payload) ? QueryBoundWitnessWrapper : BoundWitnessWrapper\n        const bw = wrapper.parse(payload)\n        if (await bw.getValid()) {\n          results[0].push(payload)\n        } else {\n          const errors = await bw.getErrors()\n          console.log(`validByType.Error: ${JSON.stringify(errors, null, 2)}`)\n        }\n      } else {\n        const payloadWrapper = PayloadWrapper.wrap(payload)\n        if (await payloadWrapper.getValid()) {\n          results[1].push(payload)\n        }\n      }\n      return\n    }),\n  )\n  return results\n}\n","export * from './Archivist.js'\nexport * from '@xyo-network/archivist-model-mongodb'\n"],"mappings":";;;;AAAA,SAASA,cAAc;AAEvB,SAASC,yBAAyB;AAElC,SAASC,4BAA4BC,gCAAgC;AACrE,SAASC,oCAAoC;AAC7C,SAASC,0BAA0B;AACnC,SAASC,sBAAsB;AAK/B,SAASC,sBAAsBC,0BAA0B;AACzD,SAASC,kBAAAA,uBAAsB;AAC/B,SAASC,gBAAgB;;;ACbzB,SAASC,gBAAgBC,2BAA2B;AACpD,SAASC,qBAAqBC,gCAAgC;AAE9D,SAASC,sBAAsB;AAExB,IAAMC,cAAc,8BAAOC,WAAsB,CAAA,MAAE;AACxD,QAAMC,UAAuC;IAAC,CAAA;IAAI,CAAA;;AAClD,QAAMC,QAAQC,IACZH,SAASI,IAAI,OAAOC,YAAAA;AAClB,QAAIC,eAAeD,OAAAA,GAAU;AAC3B,YAAME,UAAUC,oBAAoBH,OAAAA,IAAWI,2BAA2BC;AAC1E,YAAMC,KAAKJ,QAAQK,MAAMP,OAAAA;AACzB,UAAI,MAAMM,GAAGE,SAAQ,GAAI;AACvBZ,gBAAQ,CAAA,EAAGa,KAAKT,OAAAA;MAClB,OAAO;AACL,cAAMU,SAAS,MAAMJ,GAAGK,UAAS;AACjCC,gBAAQC,IAAI,sBAAsBC,KAAKC,UAAUL,QAAQ,MAAM,CAAA,CAAA,EAAI;MACrE;IACF,OAAO;AACL,YAAMM,iBAAiBC,eAAeC,KAAKlB,OAAAA;AAC3C,UAAI,MAAMgB,eAAeR,SAAQ,GAAI;AACnCZ,gBAAQ,CAAA,EAAGa,KAAKT,OAAAA;MAClB;IACF;AACA;EACF,CAAA,CAAA;AAEF,SAAOJ;AACT,GAvB2B;;;ADY3B,IAAMuB,uBAAuBC,mBAAmBC,iBAAAA;AAEzC,IAAMC,mBAAN,cAA+BH,qBAAAA;EApBtC,OAoBsCA;;;EACpC,OAAyBI,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAErCE,UAAoB;IAACC;IAA4BC;OAA6B,MAAMF;;;;;EAKnFG,qBAAqB;EAExC,MAAeC,OAAqC;AAClD,UAAMA,OAAO,MAAM,KAAKC,KAAK;MAAEC,OAAO;MAAGC,OAAO;IAAO,CAAA;AACvD,WAAOH,KAAK,CAAA,IAAKI,gBAAeC,KAAKL,KAAK,CAAA,CAAE,EAAEM,UAAUC;EAC1D;EAEA,MAAgBC,cAAcC,MAAY;AACxC,UAAMC,cAAe,MAAM,KAAKC,SAASC,QAAQ;MAAEC,QAAQJ;IAAK,CAAA;AAChE,QAAIC,aAAa;AACf,aAAOA;IACT,OAAO;AACL,YAAMI,SAAU,MAAM,KAAKC,eAAeH,QAAQ;QAAEC,QAAQJ;MAAK,CAAA;AACjE,UAAIK,QAAQ;AACV,eAAOA;MACT,OAAO;AACL,cAAMR,UAAW,MAAM,KAAKK,SAASC,QAAQ;UAAEI,OAAOP;QAAK,CAAA;AAC3D,YAAIH,SAAS;AACX,iBAAOA;QACT,OAAO;AACL,gBAAMW,KAAM,MAAM,KAAKF,eAAeH,QAAQ;YAAEI,OAAOP;UAAK,CAAA;AAC5D,iBAAOQ,MAAMV;QACf;MACF;IACF;EACF;EAEA,MAAyBW,WAAWC,QAA8C;AAChF,QAAIC,kBAAkB;SAAID;;AAE1B,UAAME,gBAAgB,MAAMC,QAAQC,IAAIH,gBAAgBI,IAAIX,CAAAA,WAAU,KAAKF,SAASC,QAAQ;MAAEC;IAAO,CAAA,CAAA,CAAA,GAAMY,OAAOC,MAAAA;AAClH,UAAMC,qBAAqB,IAAIC,IAAIP,aAAaG,IAAIlB,CAAAA,YAAWA,QAAQO,MAAM,CAAA;AAC7EO,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACkB,mBAAmBE,IAAIpB,IAAAA,CAAAA;AAEzE,UAAMqB,WAAW,MAAMR,QAAQC,IAAIH,gBAAgBI,IAAIX,CAAAA,WAAU,KAAKE,eAAeH,QAAQ;MAAEC;IAAO,CAAA,CAAA,CAAA,GAAMY,OAAOC,MAAAA;AACnH,UAAMK,gBAAgB,IAAIH,IAAIE,QAAQN,IAAIlB,CAAAA,YAAWA,QAAQO,MAAM,CAAA;AACnEO,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACsB,cAAcF,IAAIpB,IAAAA,CAAAA;AAEpE,UAAME,YAAY,MAAMW,QAAQC,IAAIH,gBAAgBI,IAAIR,CAAAA,UAAS,KAAKL,SAASC,QAAQ;MAAEI;IAAM,CAAA,CAAA,CAAA,GAAMS,OAAOC,MAAAA;AAC5G,UAAMM,iBAAiB,IAAIJ,IAAIjB,SAASa,IAAIlB,CAAAA,YAAWA,QAAQU,KAAK,CAAA;AACpEI,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACuB,eAAeH,IAAIpB,IAAAA,CAAAA;AAErE,UAAMwB,OAAO,MAAMX,QAAQC,IAAIH,gBAAgBI,IAAIR,CAAAA,UAAS,KAAKD,eAAeH,QAAQ;MAAEI;IAAM,CAAA,CAAA,CAAA,GAAMS,OAAOC,MAAAA;AAC7G,UAAMQ,YAAY,IAAIN,IAAIK,IAAIT,IAAIlB,CAAAA,YAAWA,QAAQU,KAAK,CAAA;AAC1DI,sBAAkBA,gBAAgBK,OAAOhB,CAAAA,SAAQ,CAACyB,UAAUL,IAAIpB,IAAAA,CAAAA;AAEhE,UAAM0B,gBAAgB;SAAId;SAAiBS;SAAYnB;SAAasB;;AACpE,UAAMG,SAAS,MAAMC,eAAeC,MAAMH,cAAcX,IAAIe,oBAAAA,CAAAA;AAE5D,WAAOH;EACT;EAEA,MAAyBI,cAAc7B,UAAmD;AACxF,UAAM,CAACM,IAAIwB,CAAAA,IAAK,MAAMC,YAAY/B,QAAAA;AAClC,UAAMgC,2BAA2B,MAAMrB,QAAQC,IAAIkB,EAAEjB,IAAI,CAACoB,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AACrG,QAAIF,yBAAyBI,SAAS,GAAG;AACvC,YAAMC,iBAAiB,MAAM,KAAKrC,SAASsC,WAAWN,wBAAAA;AACtD,UAAI,CAACK,eAAeE,gBAAgBF,eAAeG,kBAAkBR,yBAAyBI,OAC5F,OAAM,IAAIK,MAAM,4CAAA;IACpB;AACA,UAAMC,iCAAiC,MAAM/B,QAAQC,IAAIN,GAAGO,IAAI,CAACoB,OAAOC,UAAUC,mBAAmBF,OAAOC,KAAAA,CAAAA,CAAAA;AAC5G,QAAIQ,+BAA+BN,SAAS,GAAG;AAC7C,YAAMO,uBAAuB,MAAM,KAAKvC,eAAekC,WAAWI,8BAAAA;AAClE,UAAI,CAACC,qBAAqBJ,gBAAgBI,qBAAqBH,kBAAkBE,+BAA+BN,OAC9G,OAAM,IAAIK,MAAM,kDAAA;IACpB;AAEA,WAAO,MAAMf,eAAeC,MAAM;SAAIe;SAAmCV;MAA0BnB,IAAIe,oBAAAA,CAAAA;EACzG;EAEA,MAAyBgB,YAAYC,SAA8D;AAEjG,QAAI,EACFtD,OAAOuD,QAAQtD,MAAK,IAClBqD,WAAW;MAAEtD,OAAO;MAAIC,OAAO;IAAO;AAE1C,QAAI,CAACD,MAAOA,SAAQ;AACpB,QAAIA,QAAQ,IAAKA,SAAQ;AAEzB,QAAIC,SAAS,MAAOA,SAAQ;AAE5B,QAAIuD;AACJ,QAAID,QAAQ;AACV,YAAMnD,UAAU,MAAM,KAAKE,cAAciD,MAAAA;AAEzC,UAAInD,QAASoD,MAAKpD,QAAQqD;IAC5B,OAAO;AACLD,WAAKvD,UAAU,QAEXyD,SAASC,eAAe,CAAA,IAGxBD,SAASC,gBAAgBC,KAAKC,IAAG,IAAK,OAAU,GAAA;IACtD;AACA,QAAI,CAACL,GAAI,QAAO,CAAA;AAGhB,UAAMM,OAAO7D,UAAU,QAAQ,IAAI;AAGnC,UAAM8D,QAAQ9D,UAAU,QAAQ;MAAEwD,KAAK;QAAEO,KAAKR;MAAG;IAAE,IAAI;MAAEC,KAAK;QAAEQ,KAAKT;MAAG;IAAE;AAG1E,UAAMvB,gBAAgB,MAAM,KAAKxB,SAASyD,cAAc,CAACC,eAAAA;AACvD,aAAOA,WACJC,UAAgC;;QAE/B;UAAEC,QAAQN;QAAM;;QAEhB;UAAEO,OAAO;YAAEb,KAAKK;UAAK;QAAE;;QAEvB;UAAES,QAAQvE;QAAM;;QAEhB;UACEwE,YAAY;YACVC,MAAM,KAAKC,sBAAsBP;YACjCQ,UAAU;cACR;gBAAEN,QAAQN;cAAM;cAChB;gBAAEO,OAAO;kBAAEb,KAAKK;gBAAK;cAAE;cACvB;gBAAES,QAAQvE;cAAM;;UAEpB;QACF;;QAEA;UAAEsE,OAAO;YAAEb,KAAKK;UAAK;QAAE;;QAEvB;UAAES,QAAQvE;QAAM;OACjB,EACA4E,UAAU,KAAK/E,kBAAkB,EACjCgF,QAAO;IACZ,CAAA;AAGA,WAAO,MAAM1C,eAAeC,MAAMH,cAAcX,IAAIe,oBAAAA,CAAAA;EACtD;EAEA,MAAyByC,eAAe;AACtC,UAAM,MAAMA,aAAAA;AACZ,UAAM,KAAKC,cAAa;AACxB,WAAO;EACT;AACF;;;AEzKA,cAAc;","names":["exists","AbstractArchivist","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","MongoDBArchivistConfigSchema","MongoDBModuleMixin","PayloadBuilder","fromDbRepresentation","toDbRepresentation","PayloadWrapper","ObjectId","isBoundWitness","isQueryBoundWitness","BoundWitnessWrapper","QueryBoundWitnessWrapper","PayloadWrapper","validByType","payloads","results","Promise","all","map","payload","isBoundWitness","wrapper","isQueryBoundWitness","QueryBoundWitnessWrapper","BoundWitnessWrapper","bw","parse","getValid","push","errors","getErrors","console","log","JSON","stringify","payloadWrapper","PayloadWrapper","wrap","MongoDBArchivistBase","MongoDBModuleMixin","AbstractArchivist","MongoDBArchivist","configSchemas","MongoDBArchivistConfigSchema","defaultConfigSchema","queries","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","aggregateTimeoutMs","head","next","limit","order","PayloadWrapper","wrap","payload","undefined","findOneByHash","hash","dataPayload","payloads","findOne","_$hash","dataBw","boundWitnesses","_hash","bw","getHandler","hashes","remainingHashes","dataPayloads","Promise","all","map","filter","exists","dataPayloadsHashes","Set","has","dataBws","dataBwsHashes","payloadsHashes","bws","bwsHashes","foundPayloads","result","PayloadBuilder","build","fromDbRepresentation","insertHandler","p","validByType","payloadsWithExternalMeta","value","index","toDbRepresentation","length","payloadsResult","insertMany","acknowledged","insertedCount","Error","boundWitnessesWithExternalMeta","boundWitnessesResult","nextHandler","options","offset","id","_id","ObjectId","createFromTime","Date","now","sort","match","$gt","$lt","useCollection","collection","aggregate","$match","$sort","$limit","$unionWith","coll","boundWitnessSdkConfig","pipeline","maxTimeMS","toArray","startHandler","ensureIndexes"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/archivist-mongodb",
-  "version": "3.1.5",
+  "version": "3.1.7",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -37,32 +37,35 @@
   "module": "dist/node/index.mjs",
   "types": "dist/node/index.d.ts",
   "dependencies": {
-    "@xylabs/exists": "^4.4.9",
-    "@xylabs/hex": "^4.4.9",
-    "@xyo-network/archivist-abstract": "^3.5.1",
-    "@xyo-network/archivist-model": "^3.5.1",
-    "@xyo-network/archivist-model-mongodb": "^3.1.5",
-    "@xyo-network/boundwitness-model": "^3.5.1",
-    "@xyo-network/boundwitness-wrapper": "^3.5.1",
-    "@xyo-network/module-abstract-mongodb": "^3.1.5",
-    "@xyo-network/payload-builder": "^3.5.1",
-    "@xyo-network/payload-model": "^3.5.1",
-    "@xyo-network/payload-mongodb": "^3.1.5",
-    "@xyo-network/payload-wrapper": "^3.5.1"
+    "@xylabs/exists": "^4.4.10",
+    "@xylabs/hex": "^4.4.10",
+    "@xyo-network/archivist-abstract": "^3.5.2",
+    "@xyo-network/archivist-model": "^3.5.2",
+    "@xyo-network/archivist-model-mongodb": "^3.1.7",
+    "@xyo-network/boundwitness-model": "^3.5.2",
+    "@xyo-network/boundwitness-wrapper": "^3.5.2",
+    "@xyo-network/module-abstract-mongodb": "^3.1.7",
+    "@xyo-network/payload-builder": "^3.5.2",
+    "@xyo-network/payload-model": "^3.5.2",
+    "@xyo-network/payload-mongodb": "^3.1.7",
+    "@xyo-network/payload-wrapper": "^3.5.2",
+    "mongodb": "~6.11.0"
   },
   "devDependencies": {
-    "@xylabs/arraybuffer": "^4.4.9",
-    "@xylabs/jest-helpers": "^4.4.9",
+    "@xylabs/assert": "^4.4.10",
+    "@xylabs/delay": "^4.4.10",
+    "@xylabs/jest-helpers": "^4.4.10",
     "@xylabs/ts-scripts-yarn3": "^4.2.4",
     "@xylabs/tsconfig": "^4.2.4",
-    "@xyo-network/account": "^3.5.1",
-    "@xyo-network/account-model": "^3.5.1",
-    "@xyo-network/archivist-wrapper": "^3.5.1",
-    "@xyo-network/boundwitness-builder": "^3.5.1",
-    "@xyo-network/sdk-xyo-mongo-js": "^3.1.5",
+    "@xylabs/vitest-extended": "^4.4.10",
+    "@xyo-network/account": "^3.5.2",
+    "@xyo-network/archivist-wrapper": "^3.5.2",
+    "@xyo-network/boundwitness-builder": "^3.5.2",
+    "@xyo-network/sdk-xyo-mongo-js": "^3.1.7",
     "fake-indexeddb": "^6.0.0",
     "jest": "^29.7.0",
-    "typescript": "^5.7.2"
+    "typescript": "^5.7.2",
+    "vitest": "^2.1.8"
   },
   "publishConfig": {
     "access": "public"

package/src/Archivist.ts

@@ -1,7 +1,8 @@
 import { exists } from '@xylabs/exists'
 import type { Hash } from '@xylabs/hex'
 import { AbstractArchivist } from '@xyo-network/archivist-abstract'
-import { ArchivistInsertQuerySchema } from '@xyo-network/archivist-model'
+import type { ArchivistNextOptions } from '@xyo-network/archivist-model'
+import { ArchivistInsertQuerySchema, ArchivistNextQuerySchema } from '@xyo-network/archivist-model'
 import { MongoDBArchivistConfigSchema } from '@xyo-network/archivist-model-mongodb'
 import { MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
@@ -11,6 +12,7 @@ import type {
 import type { PayloadWithMongoMeta } from '@xyo-network/payload-mongodb'
 import { fromDbRepresentation, toDbRepresentation } from '@xyo-network/payload-mongodb'
 import { PayloadWrapper } from '@xyo-network/payload-wrapper'
+import { ObjectId } from 'mongodb'
 
 import { validByType } from './lib/index.js'
 
@@ -20,13 +22,38 @@ export class MongoDBArchivist extends MongoDBArchivistBase {
   static override readonly configSchemas: Schema[] = [...super.configSchemas, MongoDBArchivistConfigSchema]
   static override readonly defaultConfigSchema: Schema = MongoDBArchivistConfigSchema
 
-  override readonly queries: string[] = [ArchivistInsertQuerySchema, ...super.queries]
+  override readonly queries: string[] = [ArchivistInsertQuerySchema, ArchivistNextQuerySchema, ...super.queries]
+
+  /**
+   * The amount of time to allow the aggregate query to execute
+   */
+  protected readonly aggregateTimeoutMs = 10_000
 
   override async head(): Promise<Payload | undefined> {
-    const head = await (await this.payloads.find({})).sort({ _timestamp: -1 }).limit(1).toArray()
+    const head = await this.next({ limit: 1, order: 'desc' })
     return head[0] ? PayloadWrapper.wrap(head[0]).payload : undefined
   }
 
+  protected async findOneByHash(hash: Hash) {
+    const dataPayload = (await this.payloads.findOne({ _$hash: hash }))
+    if (dataPayload) {
+      return dataPayload
+    } else {
+      const dataBw = (await this.boundWitnesses.findOne({ _$hash: hash }))
+      if (dataBw) {
+        return dataBw
+      } else {
+        const payload = (await this.payloads.findOne({ _hash: hash }))
+        if (payload) {
+          return payload
+        } else {
+          const bw = (await this.boundWitnesses.findOne({ _hash: hash }))
+          return bw ?? undefined
+        }
+      }
+    }
+  }
+
   protected override async getHandler(hashes: Hash[]): Promise<WithMeta<Payload>[]> {
     let remainingHashes = [...hashes]
 
@@ -58,16 +85,82 @@ export class MongoDBArchivist extends MongoDBArchivistBase {
     if (payloadsWithExternalMeta.length > 0) {
       const payloadsResult = await this.payloads.insertMany(payloadsWithExternalMeta)
       if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length)
-        throw new Error('MongoDBDeterministicArchivist: Error inserting Payloads')
+        throw new Error('MongoDBArchivist: Error inserting Payloads')
     }
     const boundWitnessesWithExternalMeta = await Promise.all(bw.map((value, index) => toDbRepresentation(value, index)))
     if (boundWitnessesWithExternalMeta.length > 0) {
       const boundWitnessesResult = await this.boundWitnesses.insertMany(boundWitnessesWithExternalMeta)
       if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length)
-        throw new Error('MongoDBDeterministicArchivist: Error inserting BoundWitnesses')
+        throw new Error('MongoDBArchivist: Error inserting BoundWitnesses')
     }
 
-    return await PayloadBuilder.build([...boundWitnessesWithExternalMeta, ...payloadsWithExternalMeta])
+    return await PayloadBuilder.build([...boundWitnessesWithExternalMeta, ...payloadsWithExternalMeta].map(fromDbRepresentation))
+  }
+
+  protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithMeta<Payload>[]> {
+    // Sanitize inputs and set defaults
+    let {
+      limit, offset, order,
+    } = options ?? { limit: 10, order: 'desc' }
+
+    if (!limit) limit = 10
+    if (limit > 100) limit = 100
+
+    if (order != 'asc') order = 'desc'
+
+    let id: ObjectId | undefined
+    if (offset) {
+      const payload = await this.findOneByHash(offset)
+      // TODO: Should we throw an error if the requested payload is not found?
+      if (payload) id = payload._id
+    } else {
+      id = order === 'asc'
+      // If ascending, start from the beginning of time
+        ? ObjectId.createFromTime(0)
+        // If descending, start from now (plus a bit more in the future to ensure
+        // them most recent ObjectIds are included)
+        : ObjectId.createFromTime((Date.now() + 10_000) / 1000)
+    }
+    if (!id) return []
+
+    // Create aggregate criteria
+    const sort = order === 'asc' ? 1 : -1
+    // TODO: How to handle random component of ID across multiple collections
+    // to ensure we don't skip some payloads
+    const match = order === 'asc' ? { _id: { $gt: id } } : { _id: { $lt: id } }
+
+    // Run the aggregate query
+    const foundPayloads = await this.payloads.useCollection((collection) => {
+      return collection
+        .aggregate<PayloadWithMongoMeta>([
+          // Pre-filter payloads collection
+          { $match: match },
+          // Sort payloads by _id
+          { $sort: { _id: sort } },
+          // Limit payloads to the first N payloads
+          { $limit: limit },
+          // Combine with filtered boundWitnesses collection
+          {
+            $unionWith: {
+              coll: this.boundWitnessSdkConfig.collection,
+              pipeline: [
+                { $match: match }, // Pre-filter boundWitnesses
+                { $sort: { _id: sort } }, // Sort boundWitnesses by _id
+                { $limit: limit }, // Limit boundWitnesses to the first N boundWitnesses
+              ],
+            },
+          },
+          // Sort the combined result by _id
+          { $sort: { _id: sort } },
+          // Limit the final result to N documents
+          { $limit: limit },
+        ])
+        .maxTimeMS(this.aggregateTimeoutMs)
+        .toArray()
+    })
+
+    // Convert from DB representation to Payloads
+    return await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))
   }
 
   protected override async startHandler() {

package/src/lib/validByType.ts

@@ -12,7 +12,7 @@ export const validByType = async (payloads: Payload[] = []) => {
         const wrapper = isQueryBoundWitness(payload) ? QueryBoundWitnessWrapper : BoundWitnessWrapper
         const bw = wrapper.parse(payload)
         if (await bw.getValid()) {
-          results[0].push(bw.payload)
+          results[0].push(payload)
         } else {
           const errors = await bw.getErrors()
           console.log(`validByType.Error: ${JSON.stringify(errors, null, 2)}`)
@@ -20,7 +20,7 @@ export const validByType = async (payloads: Payload[] = []) => {
       } else {
         const payloadWrapper = PayloadWrapper.wrap(payload)
         if (await payloadWrapper.getValid()) {
-          results[1].push(payloadWrapper.payload)
+          results[1].push(payload)
         }
       }
       return