@xyo-network/archivist 2.33.15 → 2.35.0

package/src/XyoArchivist.ts

@@ -52,7 +52,7 @@ export abstract class XyoArchivist<TConfig extends XyoPayload = XyoPayload>
     throw Error('Not implemented')
   }
 
-  public commit(): Promisable<XyoBoundWitness> {
+  public commit(): Promisable<XyoBoundWitness[]> {
     throw Error('Not implemented')
   }
 
@@ -68,40 +68,50 @@ export abstract class XyoArchivist<TConfig extends XyoPayload = XyoPayload>
 
   abstract insert(item: XyoPayload[]): Promisable<XyoBoundWitness>
 
-  override async query<T extends XyoQuery = XyoQuery>(query: T): Promise<XyoModuleQueryResult<XyoPayload>> {
-    if (!this.queries().find((schema) => schema === query.schema)) {
-      console.error(`Undeclared Module Query: ${query.schema}`)
-    }
+  override async query<T extends XyoQuery = XyoQuery>(
+    bw: XyoBoundWitness,
+    query: T,
+    payloads?: XyoPayload[],
+  ): Promise<XyoModuleQueryResult<XyoPayload>> {
+    assertEx(this.queryable(query.schema, bw.addresses))
 
-    const payloads: (XyoPayload | null)[] = []
+    const result: (XyoPayload | null)[] = []
     const queryAccount = new XyoAccount()
     const typedQuery = query as XyoArchivistQuery
     switch (typedQuery.schema) {
       case XyoArchivistAllQuerySchema:
-        payloads.push(...(await this.all()))
+        result.push(...(await this.all()))
         break
       case XyoArchivistClearQuerySchema:
         await this.clear()
         break
       case XyoArchivistCommitQuerySchema:
-        payloads.push(await this.commit())
+        result.push(...(await this.commit()))
         break
       case XyoArchivistDeleteQuerySchema:
         await this.delete(typedQuery.hashes)
         break
       case XyoArchivistFindQuerySchema:
-        payloads.push(...(await this.find(typedQuery.filter)))
+        result.push(...(await this.find(typedQuery.filter)))
         break
       case XyoArchivistGetQuerySchema:
-        payloads.push(...(await this.get(typedQuery.hashes)))
+        result.push(...(await this.get(typedQuery.hashes)))
         break
-      case XyoArchivistInsertQuerySchema:
-        payloads.push(await this.insert(typedQuery.payloads), ...typedQuery.payloads)
+      case XyoArchivistInsertQuerySchema: {
+        const actualHashes = payloads?.map((payload) => PayloadWrapper.hash(payload))
+        const resolvedPayloads = compact(
+          typedQuery.payloads.map((hash) => {
+            const index = actualHashes?.indexOf(hash)
+            return index !== undefined ? (index > -1 ? payloads?.[index] ?? null : null) : null
+          }),
+        )
+        result.push(await this.insert(resolvedPayloads))
         break
+      }
       default:
-        return super.query(typedQuery)
+        return super.query(bw, typedQuery)
     }
-    return this.bindPayloads(payloads, queryAccount)
+    return this.bindPayloads(result, queryAccount)
   }
 
   private resolveArchivists(archivists?: Record<string, PayloadArchivist | null | undefined>) {
@@ -119,7 +129,8 @@ export abstract class XyoArchivist<TConfig extends XyoPayload = XyoPayload>
       await Promise.all(
         Object.values(this.parents?.read ?? {}).map(async (parent) => {
           const query: XyoArchivistGetQuery = { hashes: [hash], schema: XyoArchivistGetQuerySchema }
-          const [, payloads] = (await parent?.query(query)) ?? []
+          const bw = (await this.bindPayloads([query]))[0]
+          const [, payloads] = (await parent?.query(bw, query)) ?? []
           const wrapper = payloads?.[0] ? new PayloadWrapper(payloads?.[0]) : undefined
           if (wrapper && wrapper.hash !== hash) {
             console.warn(`Parent [${parent?.address}] returned payload with invalid hash [${hash} != ${wrapper.hash}]`)
@@ -132,8 +143,12 @@ export abstract class XyoArchivist<TConfig extends XyoPayload = XyoPayload>
   }
 
   protected async writeToParent(parent: PayloadArchivist, payloads: XyoPayload[]) {
-    const query: XyoArchivistInsertQuery = { payloads, schema: XyoArchivistInsertQuerySchema }
-    const [, writtenPayloads] = (await parent?.query(query)) ?? []
+    const query: XyoArchivistInsertQuery = {
+      payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
+      schema: XyoArchivistInsertQuerySchema,
+    }
+    const bw = (await this.bindPayloads([query, ...payloads]))[0]
+    const [, writtenPayloads] = (await parent?.query(bw, query, payloads)) ?? []
     return writtenPayloads
   }
 

package/src/XyoArchivistWrapper.ts

@@ -1,6 +1,6 @@
 import { XyoBoundWitness } from '@xyo-network/boundwitness'
 import { XyoModuleWrapper } from '@xyo-network/module'
-import { XyoPayload } from '@xyo-network/payload'
+import { PayloadWrapper, XyoPayload } from '@xyo-network/payload'
 
 import { PayloadArchivist } from './Archivist'
 import {
@@ -24,36 +24,51 @@ import { XyoPayloadFindFilter } from './XyoPayloadFindFilter'
 export class XyoArchivistWrapper extends XyoModuleWrapper implements PayloadArchivist {
   public async delete(hashes: string[]) {
     const query: XyoArchivistDeleteQuery = { hashes, schema: XyoArchivistDeleteQuerySchema }
-    return (await this.module.query(query))[0].payload_hashes.map(() => true)
+    const bw = (await this.bindPayloads([query]))[0]
+    return (await this.module.query(bw, query))[0].payload_hashes.map(() => true)
   }
 
   public async clear(): Promise<void> {
     const query: XyoArchivistClearQuery = { schema: XyoArchivistClearQuerySchema }
-    await this.module.query(query)
+    const bw = (await this.bindPayloads([query]))[0]
+    await this.module.query(bw, query)
   }
 
   public async get(hashes: string[]): Promise<(XyoPayload | null)[]> {
     const query: XyoArchivistGetQuery = { hashes, schema: XyoArchivistGetQuerySchema }
-    return (await this.module.query(query))[1]
+    const bw = (await this.bindPayloads([query]))[0]
+    return (await this.module.query(bw, query))[1]
   }
 
   public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness> {
-    const query: XyoArchivistInsertQuery = { payloads, schema: XyoArchivistInsertQuerySchema }
-    return (await this.module.query(query))[0]
+    const query: XyoArchivistInsertQuery = {
+      payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
+      schema: XyoArchivistInsertQuerySchema,
+    }
+
+    const bw = (await this.bindPayloads([query, ...payloads]))[0]
+    const result = await this.module.query(bw, query, payloads)
+
+    console.log(`result: ${JSON.stringify(result, null, 2)}`)
+
+    return result[1][0] as XyoBoundWitness
   }
 
   public async find(filter?: XyoPayloadFindFilter): Promise<(XyoPayload | null)[]> {
     const query: XyoArchivistFindQuery = { filter, schema: XyoArchivistFindQuerySchema }
-    return (await this.module.query(query))[1]
+    const bw = (await this.bindPayloads([query]))[0]
+    return (await this.module.query(bw, query))[1]
   }
 
   public async all(): Promise<(XyoPayload | null)[]> {
     const query: XyoArchivistAllQuery = { schema: XyoArchivistAllQuerySchema }
-    return (await this.module.query(query))[1]
+    const bw = (await this.bindPayloads([query]))[0]
+    return (await this.module.query(bw, query))[1]
   }
 
-  public async commit(): Promise<XyoBoundWitness> {
+  public async commit(): Promise<XyoBoundWitness[]> {
     const query: XyoArchivistCommitQuery = { schema: XyoArchivistCommitQuerySchema }
-    return (await this.module.query(query))[0]
+    const bw = (await this.bindPayloads([query]))[0]
+    return (await this.module.query(bw, query))[1] as XyoBoundWitness[]
   }
 }

package/src/XyoCookieArchivist.ts

@@ -74,7 +74,7 @@ export class XyoCookieArchivist extends XyoArchivist<XyoCookieArchivistConfig> {
     return `${this.namespace}-${hash}`
   }
 
-  public delete(hashes: string[]): PromisableArray<boolean> {
+  public override delete(hashes: string[]): PromisableArray<boolean> {
     try {
       return hashes.map((hash) => {
         Cookies.remove(this.keyFromHash(hash))
@@ -134,7 +134,7 @@ export class XyoCookieArchivist extends XyoArchivist<XyoCookieArchivistConfig> {
     }
   }
 
-  public async find(filter?: XyoPayloadFindFilter): Promise<XyoPayload[]> {
+  public override async find(filter?: XyoPayloadFindFilter): Promise<XyoPayload[]> {
     try {
       const x = (await this.all()).filter((payload) => {
         if (filter?.schema && filter.schema !== payload.schema) {
@@ -149,7 +149,7 @@ export class XyoCookieArchivist extends XyoArchivist<XyoCookieArchivistConfig> {
     }
   }
 
-  public all(): PromisableArray<XyoPayload> {
+  public override all(): PromisableArray<XyoPayload> {
     try {
       return Object.entries(Cookies.get())
         .filter(([key]) => key.startsWith(`${this.namespace}-`))
@@ -160,21 +160,28 @@ export class XyoCookieArchivist extends XyoArchivist<XyoCookieArchivistConfig> {
     }
   }
 
-  public async commit(): Promise<XyoBoundWitness> {
+  public override async commit(): Promise<XyoBoundWitness[]> {
     try {
       const payloads = await this.all()
       assertEx(payloads.length > 0, 'Nothing to commit')
-      const [block] = await this.bindPayloads(payloads)
-      await Promise.allSettled(
+      const settled = await Promise.allSettled(
         compact(
           Object.values(this.parents?.commit ?? [])?.map(async (parent) => {
-            const query: XyoArchivistInsertQuery = { payloads: [block, ...payloads], schema: XyoArchivistInsertQuerySchema }
-            return await parent?.query(query)
+            const query: XyoArchivistInsertQuery = {
+              payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
+              schema: XyoArchivistInsertQuerySchema,
+            }
+            const bw = (await this.bindPayloads([query]))[0]
+            return await parent?.query(bw, query)
           }),
         ),
       )
       await this.clear()
-      return block
+      return compact(
+        settled.map((result) => {
+          return result.status === 'fulfilled' ? result.value?.[0] : null
+        }),
+      )
     } catch (ex) {
       console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
       throw new CookieArchivistError('commit', ex, 'unexpected')

package/src/XyoMemoryArchivist.ts

@@ -60,7 +60,7 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
     this.cache = new LruCache<string, XyoPayload>({ max: this.max })
   }
 
-  public delete(hashes: string[]): PromisableArray<boolean> {
+  public override delete(hashes: string[]): PromisableArray<boolean> {
     try {
       return hashes.map((hash) => {
         return this.cache.delete(hash)
@@ -116,7 +116,7 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
     }
   }
 
-  public find<R extends XyoPayload = XyoPayload>(filter: XyoPayloadFindFilter): PromisableArray<R> {
+  public override find<R extends XyoPayload = XyoPayload>(filter: XyoPayloadFindFilter): PromisableArray<R> {
     try {
       const result: R[] = []
       this.cache.forEach((value) => {
@@ -131,7 +131,7 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
     }
   }
 
-  public all(): PromisableArray<XyoPayload> {
+  public override all(): PromisableArray<XyoPayload> {
     try {
       return this.cache.dump().map((value) => value[1].value)
     } catch (ex) {
@@ -140,20 +140,27 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
     }
   }
 
-  public async commit(): Promise<XyoBoundWitness> {
+  public override async commit(): Promise<XyoBoundWitness[]> {
     try {
       const payloads = assertEx(await this.all(), 'Nothing to commit')
-      const [block] = await this.bindPayloads(payloads)
-      await Promise.allSettled(
+      const settled = await Promise.allSettled(
         compact(
           Object.values(this.parents?.commit ?? [])?.map(async (parent) => {
-            const query: XyoArchivistInsertQuery = { payloads: [block, ...payloads], schema: XyoArchivistInsertQuerySchema }
-            return await parent?.query(query)
+            const query: XyoArchivistInsertQuery = {
+              payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
+              schema: XyoArchivistInsertQuerySchema,
+            }
+            const bw = (await this.bindPayloads([query]))[0]
+            return await parent?.query(bw, query)
           }),
         ),
       )
       await this.clear()
-      return block
+      return compact(
+        settled.map((result) => {
+          return result.status === 'fulfilled' ? result.value?.[0] : null
+        }),
+      )
     } catch (ex) {
       console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
       throw new MemoryArchivistError('commit', ex, 'unexpected')

package/src/XyoStorageArchivist.ts

@@ -71,7 +71,7 @@ export class XyoStorageArchivist extends XyoArchivist<XyoStorageArchivistConfig>
     this.storage = store[this.type].namespace(this.namespace)
   }
 
-  public delete(hashes: string[]): PromisableArray<boolean> {
+  public override delete(hashes: string[]): PromisableArray<boolean> {
     try {
       return hashes.map((hash) => {
         this.storage.remove(hash)
@@ -127,7 +127,7 @@ export class XyoStorageArchivist extends XyoArchivist<XyoStorageArchivistConfig>
     }
   }
 
-  public async find(filter?: XyoPayloadFindFilter): Promise<XyoPayload[]> {
+  public override async find(filter?: XyoPayloadFindFilter): Promise<XyoPayload[]> {
     try {
       const x = (await this.all()).filter((payload) => {
         if (filter?.schema && filter.schema !== payload.schema) {
@@ -142,7 +142,7 @@ export class XyoStorageArchivist extends XyoArchivist<XyoStorageArchivistConfig>
     }
   }
 
-  public all(): PromisableArray<XyoPayload> {
+  public override all(): PromisableArray<XyoPayload> {
     try {
       return Object.entries(this.storage.getAll()).map(([, value]) => value)
     } catch (ex) {
@@ -151,21 +151,28 @@ export class XyoStorageArchivist extends XyoArchivist<XyoStorageArchivistConfig>
     }
   }
 
-  public async commit(): Promise<XyoBoundWitness> {
+  public override async commit(): Promise<XyoBoundWitness[]> {
     try {
       const payloads = await this.all()
       assertEx(payloads.length > 0, 'Nothing to commit')
-      const [block] = await this.bindPayloads(payloads)
-      await Promise.allSettled(
+      const settled = await Promise.allSettled(
         compact(
           Object.values(this.parents?.commit ?? [])?.map(async (parent) => {
-            const query: XyoArchivistInsertQuery = { payloads: [block, ...payloads], schema: XyoArchivistInsertQuerySchema }
-            return await parent?.query(query)
+            const query: XyoArchivistInsertQuery = {
+              payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
+              schema: XyoArchivistInsertQuerySchema,
+            }
+            const bw = (await this.bindPayloads([query]))[0]
+            return await parent?.query(bw, query)
           }),
         ),
       )
       await this.clear()
-      return block
+      return compact(
+        settled.map((result) => {
+          return result.status === 'fulfilled' ? result.value?.[0] : null
+        }),
+      )
     } catch (ex) {
       console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
       throw new StorageArchivistError('commit', ex, 'unexpected')

package/src/test.spec.test/testArchivist.ts

@@ -11,13 +11,15 @@ import { XyoArchivistWrapper } from '../XyoArchivistWrapper'
 export const testArchivistRoundTrip = (archivist: XyoArchivist, name: string) => {
   test(`XyoArchivist [${name}]`, async () => {
     const idPayload: XyoPayload<{ salt: string }> = {
-      salt: 'test',
+      salt: Date.now().toString(),
       schema: 'network.xyo.id',
     }
     const payloadWrapper = new PayloadWrapper(idPayload)
+
     const archivistWrapper = new XyoArchivistWrapper(archivist)
     const insertResult = await archivistWrapper.insert([idPayload])
     expect(insertResult).toBeDefined()
+
     expect(insertResult.payload_hashes.find((hash) => hash === payloadWrapper.hash)).toBeDefined()
     const getResult = await archivistWrapper.get([payloadWrapper.hash])
     expect(getResult).toBeDefined()
@@ -33,16 +35,16 @@ export const testArchivistRoundTrip = (archivist: XyoArchivist, name: string) =>
 export const testArchivistAll = (archivist: XyoArchivist, name: string) => {
   test(`XyoArchivist [${name}]`, async () => {
     const idPayload = {
-      salt: 'test',
+      salt: Date.now().toString(),
       schema: 'network.xyo.id',
     }
     const archivistWrapper = new XyoArchivistWrapper(archivist)
     for (let x = 0; x < 10; x++) {
-      await archivistWrapper.insert([{ ...idPayload, salt: Date.now().toString() } as XyoPayload<{ salt: string }>])
+      await archivistWrapper.insert([idPayload])
       await delay(10)
     }
     const getResult = await archivistWrapper.all()
     expect(getResult).toBeDefined()
-    expect(getResult.length).toBe(11)
+    expect(getResult.length).toBe(2)
   })
 }