@xyo-network/archivist 2.35.0 → 2.36.0-rc.1

package/package.json

@@ -10,11 +10,11 @@
   },
   "dependencies": {
     "@xylabs/sdk-js": "^2.6.7",
-    "@xyo-network/account": "^2.35.0",
-    "@xyo-network/boundwitness": "^2.35.0",
-    "@xyo-network/module": "^2.35.0",
-    "@xyo-network/payload": "^2.35.0",
-    "@xyo-network/promise": "^2.35.0",
+    "@xyo-network/account": "^2.36.0-rc.1",
+    "@xyo-network/boundwitness": "^2.36.0-rc.1",
+    "@xyo-network/module": "^2.36.0-rc.1",
+    "@xyo-network/payload": "^2.36.0-rc.1",
+    "@xyo-network/promise": "^2.36.0-rc.1",
     "js-cookie": "^3.0.1",
     "lodash": "^4.17.21",
     "lru-cache": "^7.14.0",
@@ -58,5 +58,6 @@
   },
   "sideEffects": false,
   "types": "dist/esm/index.d.ts",
-  "version": "2.35.0"
+  "version": "2.36.0-rc.1",
+  "stableVersion": "2.35.0"
 }

package/src/Archivist.ts

@@ -11,7 +11,7 @@ export interface ReadArchivist<TReadResponse, TId = string> {
 }
 
 export interface WriteArchivist<TReadResponse, TWriteResponse = TReadResponse, TWrite = TReadResponse, TId = string> {
-  insert(item: TWrite[]): Promisable<TWriteResponse>
+  insert(item: TWrite[]): PromisableArray<TWriteResponse>
   delete?(ids: TId[]): PromisableArray<boolean>
   clear?(): Promisable<void>
 }

package/src/XyoArchivist.ts

@@ -1,8 +1,15 @@
 import { assertEx } from '@xylabs/sdk-js'
 import { XyoAccount } from '@xyo-network/account'
 import { XyoBoundWitness } from '@xyo-network/boundwitness'
-import { XyoModule, XyoModuleInitializeQuerySchema, XyoModuleQueryResult, XyoModuleShutdownQuerySchema, XyoQuery } from '@xyo-network/module'
-import { PayloadWrapper, XyoPayload } from '@xyo-network/payload'
+import {
+  ModuleQueryResult,
+  QueryBoundWitnessWrapper,
+  XyoModule,
+  XyoModuleInitializeQuerySchema,
+  XyoModuleShutdownQuerySchema,
+  XyoQueryBoundWitness,
+} from '@xyo-network/module'
+import { PayloadWrapper, XyoPayload, XyoPayloads } from '@xyo-network/payload'
 import { NullablePromisableArray, Promisable, PromisableArray } from '@xyo-network/promise'
 import compact from 'lodash/compact'
 
@@ -66,52 +73,49 @@ export abstract class XyoArchivist<TConfig extends XyoPayload = XyoPayload>
 
   abstract get(hashes: string[]): NullablePromisableArray<XyoPayload>
 
-  abstract insert(item: XyoPayload[]): Promisable<XyoBoundWitness>
+  abstract insert(item: XyoPayload[]): PromisableArray<XyoBoundWitness>
 
-  override async query<T extends XyoQuery = XyoQuery>(
-    bw: XyoBoundWitness,
+  override async query<T extends XyoQueryBoundWitness = XyoQueryBoundWitness>(
     query: T,
-    payloads?: XyoPayload[],
-  ): Promise<XyoModuleQueryResult<XyoPayload>> {
-    assertEx(this.queryable(query.schema, bw.addresses))
+    payloads?: XyoPayloads,
+  ): Promise<ModuleQueryResult<XyoPayload>> {
+    const wrapper = QueryBoundWitnessWrapper.parseQuery<XyoArchivistQuery>(query, payloads)
+    const typedQuery = wrapper.query.payload
+    assertEx(this.queryable(typedQuery.schema, wrapper.addresses))
 
-    const result: (XyoPayload | null)[] = []
+    const resultPayloads: (XyoPayload | null)[] = []
     const queryAccount = new XyoAccount()
-    const typedQuery = query as XyoArchivistQuery
     switch (typedQuery.schema) {
       case XyoArchivistAllQuerySchema:
-        result.push(...(await this.all()))
+        resultPayloads.push(...(await this.all()))
         break
       case XyoArchivistClearQuerySchema:
         await this.clear()
         break
       case XyoArchivistCommitQuerySchema:
-        result.push(...(await this.commit()))
+        resultPayloads.push(...(await this.commit()))
         break
       case XyoArchivistDeleteQuerySchema:
         await this.delete(typedQuery.hashes)
         break
       case XyoArchivistFindQuerySchema:
-        result.push(...(await this.find(typedQuery.filter)))
+        resultPayloads.push(...(await this.find(typedQuery.filter)))
         break
       case XyoArchivistGetQuerySchema:
-        result.push(...(await this.get(typedQuery.hashes)))
+        resultPayloads.push(...(await this.get(typedQuery.hashes)))
         break
       case XyoArchivistInsertQuerySchema: {
-        const actualHashes = payloads?.map((payload) => PayloadWrapper.hash(payload))
-        const resolvedPayloads = compact(
-          typedQuery.payloads.map((hash) => {
-            const index = actualHashes?.indexOf(hash)
-            return index !== undefined ? (index > -1 ? payloads?.[index] ?? null : null) : null
-          }),
-        )
-        result.push(await this.insert(resolvedPayloads))
+        const wrappers = payloads?.map((payload) => PayloadWrapper.parse(payload)) ?? []
+        assertEx(typedQuery.payloads, `Missing payloads: ${JSON.stringify(typedQuery, null, 2)}`)
+        const resolvedWrappers = wrappers.filter((wrapper) => typedQuery.payloads.includes(wrapper.hash))
+        assertEx(resolvedWrappers.length === typedQuery.payloads.length, 'Could not find some passed hashes')
+        resultPayloads.push(...(await this.insert(resolvedWrappers.map((wrapper) => wrapper.payload))))
         break
       }
       default:
-        return super.query(bw, typedQuery)
+        return super.query(query)
     }
-    return this.bindPayloads(result, queryAccount)
+    return this.bindResult(resultPayloads, queryAccount)
   }
 
   private resolveArchivists(archivists?: Record<string, PayloadArchivist | null | undefined>) {
@@ -128,9 +132,9 @@ export abstract class XyoArchivist<TConfig extends XyoPayload = XyoPayload>
     return compact(
       await Promise.all(
         Object.values(this.parents?.read ?? {}).map(async (parent) => {
-          const query: XyoArchivistGetQuery = { hashes: [hash], schema: XyoArchivistGetQuerySchema }
-          const bw = (await this.bindPayloads([query]))[0]
-          const [, payloads] = (await parent?.query(bw, query)) ?? []
+          const queryPayload = PayloadWrapper.parse<XyoArchivistGetQuery>({ hashes: [hash], schema: XyoArchivistGetQuerySchema })
+          const query = await this.bindQuery([queryPayload.body], queryPayload.hash)
+          const [, payloads] = (await parent?.query(query[0], query[1])) ?? []
           const wrapper = payloads?.[0] ? new PayloadWrapper(payloads?.[0]) : undefined
           if (wrapper && wrapper.hash !== hash) {
             console.warn(`Parent [${parent?.address}] returned payload with invalid hash [${hash} != ${wrapper.hash}]`)
@@ -143,20 +147,22 @@ export abstract class XyoArchivist<TConfig extends XyoPayload = XyoPayload>
   }
 
   protected async writeToParent(parent: PayloadArchivist, payloads: XyoPayload[]) {
-    const query: XyoArchivistInsertQuery = {
+    const queryPayload = PayloadWrapper.parse<XyoArchivistInsertQuery>({
       payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
       schema: XyoArchivistInsertQuerySchema,
-    }
-    const bw = (await this.bindPayloads([query, ...payloads]))[0]
-    const [, writtenPayloads] = (await parent?.query(bw, query, payloads)) ?? []
-    return writtenPayloads
+    })
+    const query = await this.bindQuery([queryPayload.body], queryPayload.hash)
+    const result = (await parent?.query(query[0], query[1])) ?? []
+    return result[0]
   }
 
-  protected async writeToParents(payloads: XyoPayload[]) {
-    return await Promise.all(
-      Object.values(this.parents?.write ?? {}).map(async (parent) => {
-        return parent ? await this.writeToParent(parent, payloads) : undefined
-      }),
+  protected async writeToParents(payloads: XyoPayload[]): Promise<XyoBoundWitness[]> {
+    return compact(
+      await Promise.all(
+        Object.values(this.parents?.write ?? {}).map(async (parent) => {
+          return parent ? await this.writeToParent(parent, payloads) : undefined
+        }),
+      ),
     )
   }
 

package/src/XyoArchivistWrapper.ts

@@ -1,4 +1,4 @@
-import { XyoBoundWitness } from '@xyo-network/boundwitness'
+import { XyoBoundWitness, XyoBoundWitnessSchema } from '@xyo-network/boundwitness'
 import { XyoModuleWrapper } from '@xyo-network/module'
 import { PayloadWrapper, XyoPayload } from '@xyo-network/payload'
 
@@ -23,52 +23,51 @@ import { XyoPayloadFindFilter } from './XyoPayloadFindFilter'
 
 export class XyoArchivistWrapper extends XyoModuleWrapper implements PayloadArchivist {
   public async delete(hashes: string[]) {
-    const query: XyoArchivistDeleteQuery = { hashes, schema: XyoArchivistDeleteQuerySchema }
-    const bw = (await this.bindPayloads([query]))[0]
-    return (await this.module.query(bw, query))[0].payload_hashes.map(() => true)
+    const queryPayload = PayloadWrapper.parse<XyoArchivistDeleteQuery>({ hashes, schema: XyoArchivistDeleteQuerySchema })
+    const query = await this.bindQuery([queryPayload.body], queryPayload.hash)
+    await this.module.query(query[0], query[1])
+    return (await this.module.query(query[0], query[1]))[0].payload_hashes.map(() => true)
   }
 
   public async clear(): Promise<void> {
-    const query: XyoArchivistClearQuery = { schema: XyoArchivistClearQuerySchema }
-    const bw = (await this.bindPayloads([query]))[0]
-    await this.module.query(bw, query)
+    const queryPayload = PayloadWrapper.parse<XyoArchivistClearQuery>({ schema: XyoArchivistClearQuerySchema })
+    const query = await this.bindQuery([queryPayload.body], queryPayload.hash)
+    await this.module.query(query[0], query[1])
   }
 
   public async get(hashes: string[]): Promise<(XyoPayload | null)[]> {
-    const query: XyoArchivistGetQuery = { hashes, schema: XyoArchivistGetQuerySchema }
-    const bw = (await this.bindPayloads([query]))[0]
-    return (await this.module.query(bw, query))[1]
+    const queryPayload = PayloadWrapper.parse<XyoArchivistGetQuery>({ hashes, schema: XyoArchivistGetQuerySchema })
+    const query = await this.bindQuery([queryPayload.body], queryPayload.hash)
+    return (await this.module.query(query[0], query[1]))[1]
   }
 
-  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness> {
-    const query: XyoArchivistInsertQuery = {
+  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness[]> {
+    const queryPayload = PayloadWrapper.parse<XyoArchivistInsertQuery>({
       payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
       schema: XyoArchivistInsertQuerySchema,
-    }
-
-    const bw = (await this.bindPayloads([query, ...payloads]))[0]
-    const result = await this.module.query(bw, query, payloads)
-
-    console.log(`result: ${JSON.stringify(result, null, 2)}`)
-
-    return result[1][0] as XyoBoundWitness
+    })
+    const query = await this.bindQuery([queryPayload.payload, ...payloads], queryPayload.hash)
+    const result = await this.module.query(query[0], [queryPayload.payload, ...payloads])
+    const innerBoundWitnesses =
+      result[1]?.filter<XyoBoundWitness>((payload): payload is XyoBoundWitness => payload?.schema === XyoBoundWitnessSchema) ?? []
+    return [result[0], ...innerBoundWitnesses]
   }
 
   public async find(filter?: XyoPayloadFindFilter): Promise<(XyoPayload | null)[]> {
-    const query: XyoArchivistFindQuery = { filter, schema: XyoArchivistFindQuerySchema }
-    const bw = (await this.bindPayloads([query]))[0]
-    return (await this.module.query(bw, query))[1]
+    const queryPayload = PayloadWrapper.parse<XyoArchivistFindQuery>({ filter, schema: XyoArchivistFindQuerySchema })
+    const query = await this.bindQuery([queryPayload.payload], queryPayload.hash)
+    return (await this.module.query(query[0], [queryPayload.payload]))[1]
   }
 
   public async all(): Promise<(XyoPayload | null)[]> {
-    const query: XyoArchivistAllQuery = { schema: XyoArchivistAllQuerySchema }
-    const bw = (await this.bindPayloads([query]))[0]
-    return (await this.module.query(bw, query))[1]
+    const queryPayload = PayloadWrapper.parse<XyoArchivistAllQuery>({ schema: XyoArchivistAllQuerySchema })
+    const query = await this.bindQuery([queryPayload.body], queryPayload.hash)
+    return (await this.module.query(query[0], query[1]))[1]
   }
 
   public async commit(): Promise<XyoBoundWitness[]> {
-    const query: XyoArchivistCommitQuery = { schema: XyoArchivistCommitQuerySchema }
-    const bw = (await this.bindPayloads([query]))[0]
-    return (await this.module.query(bw, query))[1] as XyoBoundWitness[]
+    const queryPayload = PayloadWrapper.parse<XyoArchivistCommitQuery>({ schema: XyoArchivistCommitQuerySchema })
+    const query = await this.bindQuery([queryPayload.body], queryPayload.hash)
+    return (await this.module.query(query[0], query[1]))[1] as XyoBoundWitness[]
   }
 }

package/src/XyoCookieArchivist.ts

@@ -113,7 +113,7 @@ export class XyoCookieArchivist extends XyoArchivist<XyoCookieArchivistConfig> {
     }
   }
 
-  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness> {
+  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness[]> {
     try {
       const storedPayloads: XyoPayload[] = payloads.map((payload) => {
         const wrapper = new PayloadWrapper(payload)
@@ -123,11 +123,13 @@ export class XyoCookieArchivist extends XyoArchivist<XyoCookieArchivistConfig> {
         Cookies.set(key, JSON.stringify(wrapper.payload))
         return wrapper.payload
       })
-      const [boundwitness] = await this.bindPayloads(storedPayloads)
+      const result = await this.bindResult([...storedPayloads])
+      const parentBoundWitnesses: XyoBoundWitness[] = []
       if (this.writeThrough) {
-        await this.writeToParents([boundwitness, ...storedPayloads])
+        //we store the child bw also
+        parentBoundWitnesses.push(...(await this.writeToParents([result[0], ...storedPayloads])))
       }
-      return boundwitness
+      return [result[0], ...parentBoundWitnesses]
     } catch (ex) {
       console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
       throw new CookieArchivistError('insert', ex, 'unexpected')
@@ -167,19 +169,19 @@ export class XyoCookieArchivist extends XyoArchivist<XyoCookieArchivistConfig> {
       const settled = await Promise.allSettled(
         compact(
           Object.values(this.parents?.commit ?? [])?.map(async (parent) => {
-            const query: XyoArchivistInsertQuery = {
+            const queryPayload = PayloadWrapper.parse<XyoArchivistInsertQuery>({
               payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
               schema: XyoArchivistInsertQuerySchema,
-            }
-            const bw = (await this.bindPayloads([query]))[0]
-            return await parent?.query(bw, query)
+            })
+            const query = await this.bindQuery([queryPayload.body], queryPayload.hash)
+            return (await parent?.query(query[0], query[1]))?.[0]
           }),
         ),
       )
       await this.clear()
       return compact(
         settled.map((result) => {
-          return result.status === 'fulfilled' ? result.value?.[0] : null
+          return result.status === 'fulfilled' ? result.value : null
         }),
       )
     } catch (ex) {

package/src/XyoMemoryArchivist.ts

@@ -97,19 +97,22 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
     }
   }
 
-  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness> {
+  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness[]> {
     try {
       payloads.map((payload) => {
         const wrapper = new PayloadWrapper(payload)
-        const payloadWithmeta = { ...payload, _hash: wrapper.hash, _timestamp: Date.now() }
-        this.cache.set(payloadWithmeta._hash, payloadWithmeta)
-        return payloadWithmeta
+        const payloadWithMeta = { ...payload, _hash: wrapper.hash, _timestamp: Date.now() }
+        this.cache.set(payloadWithMeta._hash, payloadWithMeta)
+        return payloadWithMeta
       })
-      const [boundwitness] = await this.bindPayloads(payloads)
+
+      const result = await this.bindResult([...payloads])
+      const parentBoundWitnesses: XyoBoundWitness[] = []
       if (this.writeThrough) {
-        await this.writeToParents(payloads)
+        //we store the child bw also
+        parentBoundWitnesses.push(...(await this.writeToParents([result[0], ...payloads])))
       }
-      return boundwitness
+      return [result[0], ...parentBoundWitnesses]
     } catch (ex) {
       console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
       throw new MemoryArchivistError('insert', ex, 'unexpected')
@@ -146,19 +149,19 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
       const settled = await Promise.allSettled(
         compact(
           Object.values(this.parents?.commit ?? [])?.map(async (parent) => {
-            const query: XyoArchivistInsertQuery = {
+            const queryPayload = PayloadWrapper.parse<XyoArchivistInsertQuery>({
               payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
               schema: XyoArchivistInsertQuerySchema,
-            }
-            const bw = (await this.bindPayloads([query]))[0]
-            return await parent?.query(bw, query)
+            })
+            const query = await this.bindQuery([queryPayload.body], queryPayload.hash)
+            return (await parent?.query(query[0], query[1]))?.[0]
           }),
         ),
       )
       await this.clear()
       return compact(
         settled.map((result) => {
-          return result.status === 'fulfilled' ? result.value?.[0] : null
+          return result.status === 'fulfilled' ? result.value : null
         }),
       )
     } catch (ex) {

package/src/XyoStorageArchivist.ts

@@ -106,7 +106,7 @@ export class XyoStorageArchivist extends XyoArchivist<XyoStorageArchivistConfig>
     }
   }
 
-  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness> {
+  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness[]> {
     try {
       const storedPayloads = payloads.map((payload) => {
         const wrapper = new PayloadWrapper(payload)
@@ -116,13 +116,15 @@ export class XyoStorageArchivist extends XyoArchivist<XyoStorageArchivistConfig>
         this.storage.set(hash, wrapper.payload)
         return wrapper.payload
       })
-      const [boundwitness] = await this.bindPayloads(storedPayloads)
+      const result = await this.bindResult([...storedPayloads])
+      const parentBoundWitnesses: XyoBoundWitness[] = []
       if (this.writeThrough) {
-        await this.writeToParents([boundwitness, ...storedPayloads])
+        //we store the child bw also
+        parentBoundWitnesses.push(...(await this.writeToParents([result[0], ...storedPayloads])))
       }
-      return boundwitness
+      return [result[0], ...parentBoundWitnesses]
     } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
+      console.error(`Error: ${ex}`)
       throw new StorageArchivistError('insert', ex, 'unexpected')
     }
   }
@@ -158,19 +160,19 @@ export class XyoStorageArchivist extends XyoArchivist<XyoStorageArchivistConfig>
       const settled = await Promise.allSettled(
         compact(
           Object.values(this.parents?.commit ?? [])?.map(async (parent) => {
-            const query: XyoArchivistInsertQuery = {
+            const queryPayload = PayloadWrapper.parse<XyoArchivistInsertQuery>({
               payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
               schema: XyoArchivistInsertQuerySchema,
-            }
-            const bw = (await this.bindPayloads([query]))[0]
-            return await parent?.query(bw, query)
+            })
+            const query = await this.bindQuery([queryPayload.body], queryPayload.hash)
+            return (await parent?.query(query[0], query[1]))?.[0]
           }),
         ),
       )
       await this.clear()
       return compact(
         settled.map((result) => {
-          return result.status === 'fulfilled' ? result.value?.[0] : null
+          return result.status === 'fulfilled' ? result.value : null
         }),
       )
     } catch (ex) {

package/src/test.spec.test/testArchivist.ts

@@ -3,13 +3,14 @@
  */
 
 import { delay } from '@xylabs/delay'
+import { BoundWitnessWrapper } from '@xyo-network/boundwitness'
 import { PayloadWrapper, XyoPayload } from '@xyo-network/payload'
 
 import { XyoArchivist } from '../XyoArchivist'
 import { XyoArchivistWrapper } from '../XyoArchivistWrapper'
 
 export const testArchivistRoundTrip = (archivist: XyoArchivist, name: string) => {
-  test(`XyoArchivist [${name}]`, async () => {
+  test(`XyoArchivist RoundTrip [${name}]`, async () => {
     const idPayload: XyoPayload<{ salt: string }> = {
       salt: Date.now().toString(),
       schema: 'network.xyo.id',
@@ -18,9 +19,11 @@ export const testArchivistRoundTrip = (archivist: XyoArchivist, name: string) =>
 
     const archivistWrapper = new XyoArchivistWrapper(archivist)
     const insertResult = await archivistWrapper.insert([idPayload])
-    expect(insertResult).toBeDefined()
+    const insertResultWrappers = insertResult.map((bw) => new BoundWitnessWrapper(bw))
+    const insertResultPayload = insertResultWrappers.pop() as BoundWitnessWrapper
+    expect(insertResultPayload).toBeDefined()
 
-    expect(insertResult.payload_hashes.find((hash) => hash === payloadWrapper.hash)).toBeDefined()
+    expect(insertResultPayload.payloadHashes.find((hash) => hash === payloadWrapper.hash)).toBeDefined()
     const getResult = await archivistWrapper.get([payloadWrapper.hash])
     expect(getResult).toBeDefined()
     expect(getResult.length).toBe(1)
@@ -33,7 +36,7 @@ export const testArchivistRoundTrip = (archivist: XyoArchivist, name: string) =>
 }
 
 export const testArchivistAll = (archivist: XyoArchivist, name: string) => {
-  test(`XyoArchivist [${name}]`, async () => {
+  test(`XyoArchivist All [${name}]`, async () => {
     const idPayload = {
       salt: Date.now().toString(),
       schema: 'network.xyo.id',