@xyo-network/archivist 2.44.1 → 2.45.0-rc.1

package/src/AbstractArchivist.ts

@@ -40,12 +40,8 @@ export abstract class AbstractArchivist<TConfig extends ArchivistConfig = Archiv
 {
   private _parents?: XyoArchivistParentWrappers
 
-  protected get cacheParentReads() {
-    return !!this.config?.cacheParentReads
-  }
-
-  protected get writeThrough() {
-    return !!this.config?.writeThrough
+  protected get storeParentReads() {
+    return !!this.config?.storeParentReads
   }
 
   public all(): PromisableArray<XyoPayload> {
@@ -74,6 +70,16 @@ export abstract class AbstractArchivist<TConfig extends ArchivistConfig = Archiv
     }
   }
 
+  public async get(hashes: string[]): Promise<XyoPayload[]> {
+    return compact(
+      await Promise.all(
+        hashes.map(async (hash) => {
+          return (await this.getFromParents(hash)) ?? null
+        }),
+      ),
+    )
+  }
+
   public override queries() {
     return [ArchivistGetQuerySchema, ...super.queries()]
   }
@@ -128,21 +134,25 @@ export abstract class AbstractArchivist<TConfig extends ArchivistConfig = Archiv
 
   protected async getFromParents(hash: string) {
     const parents = await this.parents()
-    return compact(
-      await Promise.all(
-        Object.values(parents.read ?? {}).map(async (parent) => {
-          const queryPayload = PayloadWrapper.parse<ArchivistGetQuery>({ hashes: [hash], schema: ArchivistGetQuerySchema })
-          const query = await this.bindQuery(queryPayload)
-          const [, payloads] = (await parent?.query(query[0], query[1])) ?? []
-          const wrapper = payloads?.[0] ? new PayloadWrapper(payloads?.[0]) : undefined
-          if (wrapper && wrapper.hash !== hash) {
-            console.warn(`Parent [${parent?.address}] returned payload with invalid hash [${hash} != ${wrapper.hash}]`)
-            return null
-          }
-          return wrapper?.payload
-        }),
-      ),
-    )[0]
+    if (Object.entries(parents.read ?? {}).length > 0) {
+      const results = compact(
+        await Promise.all(
+          Object.values(parents.read ?? {}).map(async (parent) => {
+            const queryPayload = PayloadWrapper.parse<ArchivistGetQuery>({ hashes: [hash], schema: ArchivistGetQuerySchema })
+            const query = await this.bindQuery(queryPayload)
+            const [, payloads] = (await parent?.query(query[0], query[1])) ?? []
+            const wrapper = payloads?.[0] ? new PayloadWrapper(payloads?.[0]) : undefined
+            if (wrapper && wrapper.hash !== hash) {
+              console.warn(`Parent [${parent?.address}] returned payload with invalid hash [${hash} != ${wrapper.hash}]`)
+              return null
+            }
+            return wrapper?.payload
+          }),
+        ),
+      )
+      return results[0]
+    }
+    return null
   }
 
   protected async parents() {
@@ -171,9 +181,10 @@ export abstract class AbstractArchivist<TConfig extends ArchivistConfig = Archiv
     ).flat()
   }
 
-  private async resolveArchivists(archivists?: string[]) {
+  private async resolveArchivists(archivists: string[] = []) {
     const resolvedWrappers: Record<string, ArchivistWrapper> = {}
-    const modules = (await this.resolver?.resolve({ address: archivists })) ?? []
+    const resolvedModules = await this.resolver?.resolve({ address: archivists })
+    const modules = resolvedModules ?? []
     modules.forEach((module) => {
       const wrapper = new ArchivistWrapper(module)
       resolvedWrappers[wrapper.address] = wrapper
@@ -181,7 +192,5 @@ export abstract class AbstractArchivist<TConfig extends ArchivistConfig = Archiv
     return resolvedWrappers
   }
 
-  abstract get(hashes: string[]): PromisableArray<XyoPayload>
-
   abstract insert(item: XyoPayload[]): PromisableArray<XyoBoundWitness>
 }

package/src/CookieArchivist.ts

@@ -142,7 +142,8 @@ export class CookieArchivist extends AbstractArchivist<CookieArchivistConfig> {
       })
       const result = await this.bindResult([...storedPayloads])
       const parentBoundWitnesses: XyoBoundWitness[] = []
-      if (this.writeThrough) {
+      const parents = await this.parents()
+      if (Object.entries(parents.write ?? {}).length) {
         //we store the child bw also
         parentBoundWitnesses.push(...(await this.writeToParents([result[0], ...storedPayloads])))
       }

package/src/MemoryArchivist.ts

@@ -31,11 +31,11 @@ export type MemoryArchivistConfig = ArchivistConfig<{
 export class MemoryArchivist<TConfig extends MemoryArchivistConfig = MemoryArchivistConfig> extends AbstractArchivist<TConfig> {
   static override configSchema = MemoryArchivistConfigSchema
 
-  private cache: LruCache<string, XyoPayload>
+  private cache: LruCache<string, XyoPayload | null>
 
   protected constructor(params: ModuleParams<TConfig>) {
     super(params)
-    this.cache = new LruCache<string, XyoPayload>({ max: this.max })
+    this.cache = new LruCache<string, XyoPayload | null>({ max: this.max })
   }
 
   public get max() {
@@ -47,94 +47,67 @@ export class MemoryArchivist<TConfig extends MemoryArchivistConfig = MemoryArchi
   }
 
   public override all(): PromisableArray<XyoPayload> {
-    try {
-      return this.cache.dump().map((value) => value[1].value)
-    } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
-      throw ex
-    }
+    return compact(this.cache.dump().map((value) => value[1].value))
   }
 
   public override clear(): void | Promise<void> {
-    try {
-      this.cache.clear()
-    } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
-      throw ex
-    }
+    this.cache.clear()
   }
 
   public override async commit(): Promise<XyoBoundWitness[]> {
-    try {
-      const payloads = assertEx(await this.all(), 'Nothing to commit')
-      const settled = await Promise.allSettled(
-        compact(
-          Object.values((await this.parents()).commit ?? [])?.map(async (parent) => {
-            const queryPayload = PayloadWrapper.parse<ArchivistInsertQuery>({
-              payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
-              schema: ArchivistInsertQuerySchema,
-            })
-            const query = await this.bindQuery(queryPayload, payloads)
-            return (await parent?.query(query[0], query[1]))?.[0]
-          }),
-        ),
-      )
-      await this.clear()
-      return compact(settled.filter(fulfilled).map((result) => result.value))
-    } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
-      throw ex
-    }
+    const payloads = assertEx(await this.all(), 'Nothing to commit')
+    const settled = await Promise.allSettled(
+      compact(
+        Object.values((await this.parents()).commit ?? [])?.map(async (parent) => {
+          const queryPayload = PayloadWrapper.parse<ArchivistInsertQuery>({
+            payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
+            schema: ArchivistInsertQuerySchema,
+          })
+          const query = await this.bindQuery(queryPayload, payloads)
+          return (await parent?.query(query[0], query[1]))?.[0]
+        }),
+      ),
+    )
+    await this.clear()
+    return compact(settled.filter(fulfilled).map((result) => result.value))
   }
 
   public override delete(hashes: string[]): PromisableArray<boolean> {
-    try {
-      return hashes.map((hash) => {
-        return this.cache.delete(hash)
-      })
-    } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
-      throw ex
-    }
+    return hashes.map((hash) => {
+      return this.cache.delete(hash)
+    })
   }
 
   public async get(hashes: string[]): Promise<XyoPayload[]> {
-    try {
-      return await Promise.all(
+    return compact(
+      await Promise.all(
         hashes.map(async (hash) => {
-          const payload = this.cache.get(hash) ?? (await this.getFromParents(hash)) ?? null
-          if (this.cacheParentReads) {
+          const payload = this.cache.get(hash) ?? (await super.get([hash]))[0] ?? null
+          if (this.storeParentReads) {
             this.cache.set(hash, payload)
           }
           return payload
         }),
-      )
-    } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
-      throw ex
-    }
+      ),
+    )
   }
 
   public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness[]> {
-    try {
-      payloads.map((payload) => {
-        const wrapper = new PayloadWrapper(payload)
-        const payloadWithMeta = { ...payload, _hash: wrapper.hash, _timestamp: Date.now() }
-        this.cache.set(payloadWithMeta._hash, payloadWithMeta)
-        return payloadWithMeta
-      })
-
-      const result = await this.bindResult([...payloads])
-      const parentBoundWitnesses: XyoBoundWitness[] = []
-      if (this.writeThrough) {
-        //we store the child bw also
-        parentBoundWitnesses.push(...(await this.writeToParents([result[0], ...payloads])))
-      }
-      return [result[0], ...parentBoundWitnesses]
-    } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
-      throw ex
+    payloads.map((payload) => {
+      const wrapper = new PayloadWrapper(payload)
+      const payloadWithMeta = { ...payload, _hash: wrapper.hash, _timestamp: Date.now() }
+      this.cache.set(payloadWithMeta._hash, payloadWithMeta)
+      return payloadWithMeta
+    })
+
+    const result = await this.bindResult([...payloads])
+    const parentBoundWitnesses: XyoBoundWitness[] = []
+    const parents = await this.parents()
+    if (Object.entries(parents.write ?? {}).length) {
+      //we store the child bw also
+      parentBoundWitnesses.push(...(await this.writeToParents([result[0], ...payloads])))
     }
+    return [result[0], ...parentBoundWitnesses]
   }
 
   public override queries() {

package/src/StorageArchivist.ts

@@ -81,100 +81,77 @@ export class XyoStorageArchivist extends AbstractArchivist<StorageArchivistConfi
 
   public override all(): PromisableArray<XyoPayload> {
     this.logger?.log(`this.storage.length: ${this.storage.length}`)
-    try {
-      return Object.entries(this.storage.getAll()).map(([, value]) => value)
-    } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
-      throw ex
-    }
+    return Object.entries(this.storage.getAll()).map(([, value]) => value)
   }
 
   public override clear(): void | Promise<void> {
     this.logger?.log(`this.storage.length: ${this.storage.length}`)
-    try {
-      this.storage.clear()
-    } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
-      throw ex
-    }
+    this.storage.clear()
   }
 
   public override async commit(): Promise<XyoBoundWitness[]> {
     this.logger?.log(`this.storage.length: ${this.storage.length}`)
-    try {
-      const payloads = await this.all()
-      assertEx(payloads.length > 0, 'Nothing to commit')
-      const settled = await Promise.allSettled(
-        compact(
-          Object.values((await this.parents()).commit ?? [])?.map(async (parent) => {
-            const queryPayload = PayloadWrapper.parse<ArchivistInsertQuery>({
-              payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
-              schema: ArchivistInsertQuerySchema,
-            })
-            const query = await this.bindQuery(queryPayload, payloads)
-            return (await parent?.query(query[0], query[1]))?.[0]
-          }),
-        ),
-      )
-      // TODO - rather than clear, delete the payloads that come back as successfully inserted
-      await this.clear()
-      return compact(settled.filter(fulfilled).map((result) => result.value))
-    } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
-      throw ex
-    }
+    const payloads = await this.all()
+    assertEx(payloads.length > 0, 'Nothing to commit')
+    const settled = await Promise.allSettled(
+      compact(
+        Object.values((await this.parents()).commit ?? [])?.map(async (parent) => {
+          const queryPayload = PayloadWrapper.parse<ArchivistInsertQuery>({
+            payloads: payloads.map((payload) => PayloadWrapper.hash(payload)),
+            schema: ArchivistInsertQuerySchema,
+          })
+          const query = await this.bindQuery(queryPayload, payloads)
+          return (await parent?.query(query[0], query[1]))?.[0]
+        }),
+      ),
+    )
+    // TODO - rather than clear, delete the payloads that come back as successfully inserted
+    await this.clear()
+    return compact(settled.filter(fulfilled).map((result) => result.value))
   }
 
   public override delete(hashes: string[]): PromisableArray<boolean> {
     this.logger?.log(`hashes.length: ${hashes.length}`)
-    try {
-      return hashes.map((hash) => {
-        this.storage.remove(hash)
-        return true
-      })
-    } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
-      throw ex
-    }
+    return hashes.map((hash) => {
+      this.storage.remove(hash)
+      return true
+    })
   }
 
   public async get(hashes: string[]): Promise<XyoPayload[]> {
     this.logger?.log(`hashes.length: ${hashes.length}`)
-    try {
-      return await Promise.all(
-        hashes.map(async (hash) => {
-          const value = this.storage.get(hash)
-          return value ?? (await this.getFromParents(hash)) ?? null
-        }),
-      )
-    } catch (ex) {
-      console.error(`Error: ${JSON.stringify(ex, null, 2)}`)
-      throw ex
-    }
+
+    return await Promise.all(
+      hashes.map(async (hash) => {
+        const payload = this.storage.get(hash) ?? (await super.get([hash]))[0] ?? null
+        if (this.storeParentReads) {
+          this.storage.set(hash, payload)
+        }
+        return payload
+      }),
+    )
   }
 
   public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness[]> {
     this.logger?.log(`payloads.length: ${payloads.length}`)
-    try {
-      const storedPayloads = payloads.map((payload) => {
-        const wrapper = new PayloadWrapper(payload)
-        const hash = wrapper.hash
-        const value = JSON.stringify(wrapper.payload)
-        assertEx(value.length < this.maxEntrySize, `Payload too large [${wrapper.hash}, ${value.length}]`)
-        this.storage.set(hash, wrapper.payload)
-        return wrapper.payload
-      })
-      const result = await this.bindResult([...storedPayloads])
-      const parentBoundWitnesses: XyoBoundWitness[] = []
-      if (this.writeThrough) {
-        //we store the child bw also
-        parentBoundWitnesses.push(...(await this.writeToParents([result[0], ...storedPayloads])))
-      }
-      return [result[0], ...parentBoundWitnesses]
-    } catch (ex) {
-      console.error(`Error: ${ex}`)
-      throw ex
+
+    const storedPayloads = payloads.map((payload) => {
+      const wrapper = new PayloadWrapper(payload)
+      const hash = wrapper.hash
+      const value = JSON.stringify(wrapper.payload)
+      assertEx(value.length < this.maxEntrySize, `Payload too large [${wrapper.hash}, ${value.length}]`)
+      this.storage.set(hash, wrapper.payload)
+      return wrapper.payload
+    })
+    const [storageBoundWitness] = await this.bindResult([...storedPayloads])
+    const parentBoundWitnesses: XyoBoundWitness[] = []
+    const parents = await this.parents()
+    if (Object.entries(parents.write ?? {}).length) {
+      //we store the child bw also
+      const [parentBoundWitness] = await this.writeToParents([storageBoundWitness, ...storedPayloads])
+      parentBoundWitnesses.push(parentBoundWitness)
     }
+    return [storageBoundWitness, ...parentBoundWitnesses]
   }
 
   public override queries() {

package/src/spec/StorageArchivist.spec.ts

@@ -3,7 +3,7 @@
  */
 
 import { Account } from '@xyo-network/account'
-import { SimpleModuleResolver } from '@xyo-network/module'
+import { CompositeModuleResolver } from '@xyo-network/module'
 import { PayloadWrapper } from '@xyo-network/payload-wrapper'
 
 import { MemoryArchivist } from '../MemoryArchivist'
@@ -76,14 +76,21 @@ test('XyoArchivist Parent Write Through', async () => {
       schema: StorageArchivistConfigSchema,
       type: 'local',
     },
-    resolver: new SimpleModuleResolver().add(memory),
+    resolver: new CompositeModuleResolver().add(memory),
   })
-  await storage.start()
+  expect(await storage.start()).toBeDefined()
 
   const wrapper = new PayloadWrapper({ schema: 'network.xyo.test' })
 
-  await storage.insert([wrapper.payload])
+  expect(wrapper).toBeDefined()
 
-  expect((await storage.get([wrapper.hash])).length).toBe(1)
-  expect((await memory.get([wrapper.hash])).length).toBe(1)
+  const inserted = await storage.insert([wrapper.payload])
+
+  expect(inserted).toBeArrayOfSize(2)
+
+  const fromStorage = await storage.get([wrapper.hash])
+  const fromMemory = await memory.get([wrapper.hash])
+
+  expect(fromStorage).toBeArrayOfSize(1)
+  expect(fromMemory).toBeArrayOfSize(1)
 })