@xyo-network/archivist-mongodb 3.1.7 → 3.2.0-rc.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/archivist-mongodb",
-  "version": "3.1.7",
+  "version": "3.2.0-rc.1",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -37,37 +37,35 @@
   "module": "dist/node/index.mjs",
   "types": "dist/node/index.d.ts",
   "dependencies": {
-    "@xylabs/exists": "^4.4.10",
-    "@xylabs/hex": "^4.4.10",
-    "@xyo-network/archivist-abstract": "^3.5.2",
-    "@xyo-network/archivist-model": "^3.5.2",
-    "@xyo-network/archivist-model-mongodb": "^3.1.7",
-    "@xyo-network/boundwitness-model": "^3.5.2",
-    "@xyo-network/boundwitness-wrapper": "^3.5.2",
-    "@xyo-network/module-abstract-mongodb": "^3.1.7",
-    "@xyo-network/payload-builder": "^3.5.2",
-    "@xyo-network/payload-model": "^3.5.2",
-    "@xyo-network/payload-mongodb": "^3.1.7",
-    "@xyo-network/payload-wrapper": "^3.5.2",
-    "mongodb": "~6.11.0"
+    "@xylabs/exists": "^4.4.25",
+    "@xylabs/hex": "^4.4.25",
+    "@xyo-network/archivist-abstract": "^3.6.0-rc.16",
+    "@xyo-network/archivist-model": "^3.6.0-rc.16",
+    "@xyo-network/archivist-model-mongodb": "^3.2.0-rc.1",
+    "@xyo-network/boundwitness-model": "^3.6.0-rc.16",
+    "@xyo-network/boundwitness-wrapper": "^3.6.0-rc.16",
+    "@xyo-network/module-abstract-mongodb": "^3.2.0-rc.1",
+    "@xyo-network/payload-builder": "^3.6.0-rc.16",
+    "@xyo-network/payload-model": "^3.6.0-rc.16",
+    "@xyo-network/payload-mongodb": "^3.2.0-rc.1",
+    "@xyo-network/payload-wrapper": "^3.6.0-rc.16",
+    "mongodb": "~6.12.0"
   },
   "devDependencies": {
-    "@xylabs/assert": "^4.4.10",
-    "@xylabs/delay": "^4.4.10",
-    "@xylabs/jest-helpers": "^4.4.10",
-    "@xylabs/ts-scripts-yarn3": "^4.2.4",
-    "@xylabs/tsconfig": "^4.2.4",
-    "@xylabs/vitest-extended": "^4.4.10",
-    "@xyo-network/account": "^3.5.2",
-    "@xyo-network/archivist-wrapper": "^3.5.2",
-    "@xyo-network/boundwitness-builder": "^3.5.2",
-    "@xyo-network/sdk-xyo-mongo-js": "^3.1.7",
+    "@xylabs/delay": "^4.4.25",
+    "@xylabs/ts-scripts-yarn3": "^4.2.6",
+    "@xylabs/tsconfig": "^4.2.6",
+    "@xylabs/vitest-extended": "^4.4.25",
+    "@xyo-network/account": "^3.6.0-rc.16",
+    "@xyo-network/archivist-wrapper": "^3.6.0-rc.16",
+    "@xyo-network/boundwitness-builder": "^3.6.0-rc.16",
+    "@xyo-network/sdk-xyo-mongo-js": "^3.2.0-rc.1",
     "fake-indexeddb": "^6.0.0",
-    "jest": "^29.7.0",
     "typescript": "^5.7.2",
     "vitest": "^2.1.8"
   },
   "publishConfig": {
     "access": "public"
-  }
+  },
+  "stableVersion": "3.1.7"
 }

package/src/Archivist.ts

@@ -7,11 +7,10 @@ import { MongoDBArchivistConfigSchema } from '@xyo-network/archivist-model-mongo
 import { MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
 import type {
-  Payload, Schema, WithMeta,
+  Payload, Schema, Sequence, WithStorageMeta,
 } from '@xyo-network/payload-model'
 import type { PayloadWithMongoMeta } from '@xyo-network/payload-mongodb'
 import { fromDbRepresentation, toDbRepresentation } from '@xyo-network/payload-mongodb'
-import { PayloadWrapper } from '@xyo-network/payload-wrapper'
 import { ObjectId } from 'mongodb'
 
 import { validByType } from './lib/index.js'
@@ -29,11 +28,6 @@ export class MongoDBArchivist extends MongoDBArchivistBase {
    */
   protected readonly aggregateTimeoutMs = 10_000
 
-  override async head(): Promise<Payload | undefined> {
-    const head = await this.next({ limit: 1, order: 'desc' })
-    return head[0] ? PayloadWrapper.wrap(head[0]).payload : undefined
-  }
-
   protected async findOneByHash(hash: Hash) {
     const dataPayload = (await this.payloads.findOne({ _$hash: hash }))
     if (dataPayload) {
@@ -54,15 +48,25 @@ export class MongoDBArchivist extends MongoDBArchivistBase {
     }
   }
 
-  protected override async getHandler(hashes: Hash[]): Promise<WithMeta<Payload>[]> {
+  protected async findOneBySequence(sequence: Sequence) {
+    const dataPayload = (await this.payloads.findOne({ _sequence: sequence }))
+    if (dataPayload) {
+      return dataPayload
+    } else {
+      const dataBw = (await this.boundWitnesses.findOne({ _sequence: sequence }))
+      return dataBw
+    }
+  }
+
+  protected override async getHandler(hashes: Hash[]): Promise<WithStorageMeta<Payload>[]> {
     let remainingHashes = [...hashes]
 
-    const dataPayloads = (await Promise.all(remainingHashes.map(_$hash => this.payloads.findOne({ _$hash })))).filter(exists)
-    const dataPayloadsHashes = new Set(dataPayloads.map(payload => payload._$hash))
+    const dataPayloads = (await Promise.all(remainingHashes.map(_dataHash => this.payloads.findOne({ _dataHash })))).filter(exists)
+    const dataPayloadsHashes = new Set(dataPayloads.map(payload => payload._dataHash))
     remainingHashes = remainingHashes.filter(hash => !dataPayloadsHashes.has(hash))
 
-    const dataBws = (await Promise.all(remainingHashes.map(_$hash => this.boundWitnesses.findOne({ _$hash })))).filter(exists)
-    const dataBwsHashes = new Set(dataBws.map(payload => payload._$hash))
+    const dataBws = (await Promise.all(remainingHashes.map(_dataHash => this.boundWitnesses.findOne({ _dataHash })))).filter(exists)
+    const dataBwsHashes = new Set(dataBws.map(payload => payload._dataHash))
     remainingHashes = remainingHashes.filter(hash => !dataBwsHashes.has(hash))
 
     const payloads = (await Promise.all(remainingHashes.map(_hash => this.payloads.findOne({ _hash })))).filter(exists)
@@ -73,34 +77,34 @@ export class MongoDBArchivist extends MongoDBArchivistBase {
     const bwsHashes = new Set(bws.map(payload => payload._hash))
     remainingHashes = remainingHashes.filter(hash => !bwsHashes.has(hash))
 
-    const foundPayloads = [...dataPayloads, ...dataBws, ...payloads, ...bws] as PayloadWithMongoMeta<Payload & { _$hash: Hash; _$meta?: unknown }>[]
-    const result = await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))
+    const foundPayloads = [...dataPayloads, ...dataBws, ...payloads, ...bws] as PayloadWithMongoMeta<Payload>[]
+    const result = foundPayloads.map(fromDbRepresentation)
     // console.log(`getHandler: ${JSON.stringify(hashes, null, 2)}:${JSON.stringify(result, null, 2)}`)
     return result
   }
 
-  protected override async insertHandler(payloads: Payload[]): Promise<WithMeta<Payload>[]> {
-    const [bw, p] = await validByType(payloads)
-    const payloadsWithExternalMeta = await Promise.all(p.map((value, index) => toDbRepresentation(value, index)))
+  protected override async insertHandler(payloads: Payload[]): Promise<WithStorageMeta<Payload>[]> {
+    const [bw, p] = await validByType(await PayloadBuilder.addStorageMeta(payloads))
+    const payloadsWithExternalMeta = await Promise.all(p.map(value => toDbRepresentation(value)))
     if (payloadsWithExternalMeta.length > 0) {
       const payloadsResult = await this.payloads.insertMany(payloadsWithExternalMeta)
       if (!payloadsResult.acknowledged || payloadsResult.insertedCount !== payloadsWithExternalMeta.length)
         throw new Error('MongoDBArchivist: Error inserting Payloads')
     }
-    const boundWitnessesWithExternalMeta = await Promise.all(bw.map((value, index) => toDbRepresentation(value, index)))
+    const boundWitnessesWithExternalMeta = await Promise.all(bw.map(value => toDbRepresentation(value)))
     if (boundWitnessesWithExternalMeta.length > 0) {
       const boundWitnessesResult = await this.boundWitnesses.insertMany(boundWitnessesWithExternalMeta)
       if (!boundWitnessesResult.acknowledged || boundWitnessesResult.insertedCount !== boundWitnessesWithExternalMeta.length)
         throw new Error('MongoDBArchivist: Error inserting BoundWitnesses')
     }
 
-    return await PayloadBuilder.build([...boundWitnessesWithExternalMeta, ...payloadsWithExternalMeta].map(fromDbRepresentation))
+    return [...boundWitnessesWithExternalMeta, ...payloadsWithExternalMeta].map(fromDbRepresentation)
   }
 
-  protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithMeta<Payload>[]> {
+  protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]> {
     // Sanitize inputs and set defaults
     let {
-      limit, offset, order,
+      limit, cursor, order,
     } = options ?? { limit: 10, order: 'desc' }
 
     if (!limit) limit = 10
@@ -109,8 +113,8 @@ export class MongoDBArchivist extends MongoDBArchivistBase {
     if (order != 'asc') order = 'desc'
 
     let id: ObjectId | undefined
-    if (offset) {
-      const payload = await this.findOneByHash(offset)
+    if (cursor) {
+      const payload = await this.findOneBySequence(cursor)
       // TODO: Should we throw an error if the requested payload is not found?
       if (payload) id = payload._id
     } else {
@@ -160,7 +164,7 @@ export class MongoDBArchivist extends MongoDBArchivistBase {
     })
 
     // Convert from DB representation to Payloads
-    return await PayloadBuilder.build(foundPayloads.map(fromDbRepresentation))
+    return foundPayloads.map(fromDbRepresentation)
   }
 
   protected override async startHandler() {

package/src/lib/validByType.ts

@@ -1,11 +1,11 @@
 import type { BoundWitness } from '@xyo-network/boundwitness-model'
 import { isBoundWitness, isQueryBoundWitness } from '@xyo-network/boundwitness-model'
 import { BoundWitnessWrapper, QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-wrapper'
-import type { Payload } from '@xyo-network/payload-model'
+import type { Payload, WithStorageMeta } from '@xyo-network/payload-model'
 import { PayloadWrapper } from '@xyo-network/payload-wrapper'
 
-export const validByType = async (payloads: Payload[] = []) => {
-  const results: [BoundWitness[], Payload[]] = [[], []]
+export const validByType = async (payloads: WithStorageMeta<Payload>[] = []) => {
+  const results: [WithStorageMeta<BoundWitness>[], WithStorageMeta<Payload>[]] = [[], []]
   await Promise.all(
     payloads.map(async (payload) => {
       if (isBoundWitness(payload)) {