@xyo-network/archivist-abstract 2.91.0 → 2.91.1

package/src/AbstractArchivist.ts

@@ -2,6 +2,7 @@ import { assertEx } from '@xylabs/assert'
 import { Address, Hash } from '@xylabs/hex'
 import { compact } from '@xylabs/lodash'
 import { Promisable, PromisableArray } from '@xylabs/promise'
+import { difference } from '@xylabs/set'
 import {
   ArchivistAllQuerySchema,
   ArchivistClearQuerySchema,
@@ -12,9 +13,9 @@ import {
   ArchivistInsertQuerySchema,
   ArchivistInstance,
   ArchivistModuleEventData,
+  ArchivistNextOptions,
   ArchivistParams,
-  ArchivistQuery,
-  ArchivistQueryBase,
+  ArchivistQueries,
   asArchivistInstance,
   isArchivistInstance,
 } from '@xyo-network/archivist-model'
@@ -57,7 +58,7 @@ export abstract class AbstractArchivist<
     return this.config.requireAllParents ?? false
   }
 
-  protected override get _queryAccountPaths(): Record<ArchivistQueryBase['schema'], string> {
+  protected override get _queryAccountPaths(): Record<ArchivistQueries['schema'], string> {
     return {
       'network.xyo.query.archivist.all': '1/1',
       'network.xyo.query.archivist.clear': '1/2',
@@ -65,6 +66,7 @@ export abstract class AbstractArchivist<
       'network.xyo.query.archivist.delete': '1/4',
       'network.xyo.query.archivist.get': '1/5',
       'network.xyo.query.archivist.insert': '1/6',
+      'network.xyo.query.archivist.next': '1/7',
     }
   }
 
@@ -76,7 +78,7 @@ export abstract class AbstractArchivist<
     this._noOverride('all')
     return this.busy(async () => {
       await this.started('throw')
-      return await this.allHandler()
+      return await PayloadBuilder.build(await this.allHandler())
     })
   }
 
@@ -96,7 +98,7 @@ export abstract class AbstractArchivist<
     })
   }
 
-  async delete(hashes: Hash[]): Promise<string[]> {
+  async delete(hashes: Hash[]): Promise<Hash[]> {
     this._noOverride('delete')
     return await this.busy(async () => {
       await this.started('throw')
@@ -108,7 +110,7 @@ export abstract class AbstractArchivist<
     this._noOverride('get')
     return await this.busy(async () => {
       await this.started('throw')
-      return await this.getWithConfig(hashes)
+      return await PayloadBuilder.build(await this.getWithConfig(hashes))
     })
   }
 
@@ -121,7 +123,15 @@ export abstract class AbstractArchivist<
     })
   }
 
-  protected allHandler(): PromisableArray<WithMeta<Payload>> {
+  async next?(options?: ArchivistNextOptions): Promise<WithMeta<Payload>[]> {
+    this._noOverride('next')
+    return await this.busy(async () => {
+      await this.started('throw')
+      return await this.nextWithConfig(options)
+    })
+  }
+
+  protected allHandler(): PromisableArray<Payload> {
     throw new Error('Not implemented')
   }
 
@@ -133,11 +143,11 @@ export abstract class AbstractArchivist<
     throw new Error('Not implemented')
   }
 
-  protected deleteHandler(_hashes: string[]): PromisableArray<string> {
+  protected deleteHandler(_hashes: Hash[]): PromisableArray<Hash> {
     throw new Error('Not implemented')
   }
 
-  protected async deleteWithConfig(hashes: string[], config?: ActionConfig): Promise<string[]> {
+  protected async deleteWithConfig(hashes: Hash[], config?: ActionConfig): Promise<Hash[]> {
     const emitEvents = config?.emitEvents ?? true
 
     const deletedHashes = await this.deleteHandler(hashes)
@@ -166,13 +176,13 @@ export abstract class AbstractArchivist<
     return [foundPayloads, notfound]
   }
 
-  protected async getFromParents(hashes: Hash[]): Promise<[WithMeta<Payload>[], string[]]> {
+  protected async getFromParents(hashes: Hash[]): Promise<[WithMeta<Payload>[], Hash[]]> {
     const parents = Object.values((await this.parents())?.read ?? {})
     let remainingHashes = [...hashes]
     let parentIndex = 0
     let result: WithMeta<Payload>[] = []
 
-    //intentionally doing this serially
+    // NOTE: intentionally doing this serially
     while (parentIndex < parents.length && remainingHashes.length > 0) {
       const [found, notfound] = await this.getFromParent(remainingHashes, parents[parentIndex])
       result = [...result, ...found]
@@ -186,37 +196,59 @@ export abstract class AbstractArchivist<
     throw new Error('Not implemented')
   }
 
-  protected async getWithConfig(hashes: Hash[], config?: InsertConfig): Promise<WithMeta<Payload>[]> {
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    const emitEvents = config?.emitEvents ?? true
-    const gotten = await this.getHandler(hashes)
-    const map = await PayloadBuilder.toHashMap(gotten)
-    const dataMap = await PayloadBuilder.toDataHashMap(gotten)
-
-    const foundPayloads: WithMeta<Payload>[] = []
-    const notfoundHashes: Hash[] = []
-    for (const hash of hashes) {
-      const found = map[hash] ?? dataMap[hash]
-      if (found) {
-        foundPayloads.push(await PayloadBuilder.build<Payload>(found, true))
-      } else {
-        notfoundHashes.push(hash)
+  protected async getWithConfig(hashes: Hash[], _config?: InsertConfig): Promise<WithMeta<Payload>[]> {
+    // Filter out duplicates
+    const requestedHashes = new Set(hashes)
+
+    // Attempt to find the payloads in the store
+    const gotten = await this.getHandler([...requestedHashes])
+
+    // Do not just blindly return what the archivist told us but
+    // ensure to only return requested payloads and keep track of
+    // the ones it did not find so we can ask the parents.
+    const foundPayloads: PayloadWithMeta[] = []
+    const foundHashes = new Set<Hash>()
+
+    // NOTE: We are iterating over the returned result from the archivist
+    // (not the array of hashes passed in) to preserve the natural order of the
+    // hashes as returned by the archivist as that should loosely
+    // correspond to the order when iterated and the symmetry will
+    // be helpful for debugging
+    for (const payload of gotten) {
+      // Compute the hashes for this payload
+      const map = await PayloadBuilder.toAllHashMap([payload])
+      let requestedPayloadFound = false
+      for (const [key, payload] of Object.entries(map)) {
+        const hash = key as Hash // NOTE: Required cast as Object.entries always returns string keys
+        // If this hash was requested
+        if (requestedHashes.has(hash)) {
+          // Indicate that we found it (but do not insert it yet). Since
+          // one payload could satisfy two requested hashes (vit its dataHash
+          // & rootHash) we only want to insert that payload once but we want
+          // to keep track of all the hashes it satisfies so we can ask th
+          // parents for the ones we did not find
+          requestedPayloadFound = true
+          // Add it to the list of found hashes
+          foundHashes.add(hash)
+        }
+        if (requestedPayloadFound) foundPayloads.push(payload)
       }
     }
-
-    const [parentFoundPayloads] = await this.getFromParents(notfoundHashes)
+    // For all the hashes we did not find, ask the parents
+    const notFoundHashes = [...difference(requestedHashes, foundHashes)]
+    const [parentFoundPayloads] = await this.getFromParents(notFoundHashes)
 
     if (this.storeParentReads) {
       await this.insertWithConfig(parentFoundPayloads)
     }
-    return [...foundPayloads, ...parentFoundPayloads]
+    return await PayloadBuilder.build([...foundPayloads, ...parentFoundPayloads])
   }
 
   protected head(): Promisable<Payload | undefined> {
     return this._lastInsertedPayload
   }
 
-  protected insertHandler(_payloads: WithMeta<Payload>[]): Promise<WithMeta<Payload>[]> {
+  protected insertHandler(_payloads: Payload[]): Promise<WithMeta<Payload>[]> {
     throw new Error('Not implemented')
   }
 
@@ -241,7 +273,7 @@ export abstract class AbstractArchivist<
     const emitEvents = config?.emitEvents ?? true
     const writeToParents = config?.writeToParents ?? true
 
-    const insertedPayloads = await this.insertHandler(await PayloadBuilder.build(payloads, true))
+    const insertedPayloads = await PayloadBuilder.build(await this.insertHandler(payloads), true)
 
     if (writeToParents) {
       await this.writeToParents(insertedPayloads)
@@ -253,6 +285,15 @@ export abstract class AbstractArchivist<
     return insertedPayloads
   }
 
+  protected nextHandler(_options?: ArchivistNextOptions): Promisable<WithMeta<Payload>[]> {
+    throw new Error('Not implemented')
+  }
+
+  protected async nextWithConfig(options?: ArchivistNextOptions, _config?: InsertConfig): Promise<WithMeta<Payload>[]> {
+    const foundPayloads = await this.nextHandler(options)
+    return await PayloadBuilder.build(foundPayloads)
+  }
+
   protected async parents() {
     this._parents = this._parents ?? {
       commit: await this.resolveArchivists(this.config?.parents?.commit),
@@ -267,7 +308,7 @@ export abstract class AbstractArchivist<
     payloads: Payload[],
     queryConfig?: TConfig,
   ): Promise<ModuleQueryHandlerResult> {
-    const wrappedQuery = await QueryBoundWitnessWrapper.parseQuery<ArchivistQuery>(query, payloads)
+    const wrappedQuery = await QueryBoundWitnessWrapper.parseQuery<ArchivistQueries>(query, payloads)
     const builtQuery = await PayloadBuilder.build(query, true)
     const queryPayload = await wrappedQuery.getQuery()
     assertEx(await this.queryable(query, payloads, queryConfig))

package/src/StorageMeta.ts

@@ -16,18 +16,22 @@ export const addStorageMeta = <T extends PayloadWithMeta>(payload: T, index = 0)
   return { ...payload, _sequence: sequenceNumber(index) } as WithStorageMeta<T>
 }
 
-export const sortByStorageMeta = <T extends PayloadWithMeta>(payloads: WithStorageMeta<T>[]) => {
+export const sortByStorageMeta = <T extends PayloadWithMeta>(payloads: WithStorageMeta<T>[], direction: -1 | 1 = 1) => {
   return payloads.sort((a, b) =>
-    a._sequence < b._sequence ? -1
-    : a._sequence > b._sequence ? 1
+    a._sequence < b._sequence ? -direction
+    : a._sequence > b._sequence ? direction
     : 0,
   )
 }
 
 export function removeStorageMeta<T extends PayloadWithMeta>(payload: WithStorageMeta<T>): T
+export function removeStorageMeta<T extends PayloadWithMeta>(payloads: WithStorageMeta<T>[]): T[]
 export function removeStorageMeta<T extends PayloadWithMeta>(payload?: WithStorageMeta<T>): T | undefined
 export function removeStorageMeta<T extends PayloadWithMeta>(payload?: WithStorageMeta<T>) {
   if (!payload) return
+  if (Array.isArray(payload)) {
+    return payload.map((p) => removeStorageMeta(p))
+  }
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   const { _sequence, ...noMeta } = payload as WithStorageMeta<T>
   return noMeta as T