@xyo-network/archivist 2.32.0-rc.1 → 2.32.0-rc.4

package/src/XyoMemoryArchivist.ts

@@ -1,5 +1,5 @@
 import { assertEx } from '@xylabs/sdk-js'
-import { XyoBoundWitness, XyoBoundWitnessBuilder } from '@xyo-network/boundwitness'
+import { XyoBoundWitness } from '@xyo-network/boundwitness'
 import { XyoPayload, XyoPayloadWrapper } from '@xyo-network/payload'
 import { PromisableArray } from '@xyo-network/promisable'
 import compact from 'lodash/compact'
@@ -13,8 +13,6 @@ import {
   XyoArchivistCommitQuerySchema,
   XyoArchivistDeleteQuerySchema,
   XyoArchivistFindQuerySchema,
-  XyoArchivistGetQuery,
-  XyoArchivistGetQuerySchema,
   XyoArchivistInsertQuery,
   XyoArchivistInsertQuerySchema,
 } from './Queries'
@@ -42,14 +40,14 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
 
   private cache: LruCache<string, XyoPayload>
 
-  public override get queries() {
+  public override queries() {
     return [
-      ...super.queries,
       XyoArchivistAllQuerySchema,
       XyoArchivistDeleteQuerySchema,
       XyoArchivistClearQuerySchema,
       XyoArchivistFindQuerySchema,
       XyoArchivistCommitQuerySchema,
+      ...super.queries(),
     ]
   }
 
@@ -76,25 +74,15 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
     }
   }
 
-  protected async getFromParents(hash: string) {
-    return compact(
-      await Promise.all(
-        compact(
-          Object.values(this.parents?.read ?? {}).map(async (parent) => {
-            const query: XyoArchivistGetQuery = { hashes: [hash], schema: XyoArchivistGetQuerySchema }
-            const [, payloads] = (await parent?.query(query)) ?? []
-            return payloads?.[0]
-          }),
-        ),
-      ),
-    )[0]
-  }
-
   public async get(hashes: string[]): Promise<(XyoPayload | null)[]> {
     try {
       return await Promise.all(
         hashes.map(async (hash) => {
-          return this.cache.get(hash) ?? (await this.getFromParents(hash)) ?? null
+          const payload = this.cache.get(hash) ?? (await this.getFromParents(hash)) ?? null
+          if (this.cacheParentReads) {
+            this.cache.set(hash, payload)
+          }
+          return payload
         }),
       )
     } catch (ex) {
@@ -102,14 +90,19 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
     }
   }
 
-  public insert(payloads: XyoPayload[]): PromisableArray<XyoPayload> {
+  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness> {
     try {
-      return payloads.map((payload) => {
+      payloads.map((payload) => {
         const wrapper = new XyoPayloadWrapper(payload)
         const payloadWithmeta = { ...payload, _hash: wrapper.hash, _timestamp: Date.now() }
         this.cache.set(payloadWithmeta._hash, payloadWithmeta)
         return payloadWithmeta
       })
+      const boundwitness = this.bindPayloads(payloads)
+      if (this.writeThrough) {
+        await this.writeToParents(payloads)
+      }
+      return boundwitness
     } catch (ex) {
       throw new MemoryArchivistError('insert', ex, 'unexpected')
     }
@@ -129,7 +122,7 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
     }
   }
 
-  public all(): Promise<XyoPayload[]> | XyoPayload[] {
+  public all(): PromisableArray<XyoPayload> {
     try {
       return this.cache.dump().map((value) => value[1].value)
     } catch (ex) {
@@ -137,12 +130,10 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
     }
   }
 
-  public async commit(): Promise<XyoPayload[]> {
+  public async commit(): Promise<XyoBoundWitness> {
     try {
-      const account = assertEx(this.account, 'Account is required for commit')
       const payloads = assertEx(await this.all(), 'Nothing to commit')
-      const builder = new XyoBoundWitnessBuilder<XyoBoundWitness, XyoPayload>()
-      const block = builder.payloads(payloads).witness(account).build()
+      const block = this.bindPayloads(payloads)
       await Promise.allSettled(
         compact(
           Object.values(this.parents?.commit ?? [])?.map(async (parent) => {
@@ -152,7 +143,7 @@ export class XyoMemoryArchivist extends XyoArchivist<XyoMemoryArchivistConfig> {
         ),
       )
       await this.clear()
-      return payloads
+      return block
     } catch (ex) {
       throw new MemoryArchivistError('commit', ex, 'unexpected')
     }

package/src/XyoStorageArchivist.spec.ts

@@ -0,0 +1,14 @@
+/**
+ * @jest-environment jsdom
+ */
+
+import { testArchivistAll, testArchivistRoundTrip } from './test.spec.test'
+import { XyoStorageArchivist } from './XyoStorageArchivist'
+
+testArchivistRoundTrip(new XyoStorageArchivist({ namespace: 'test', type: 'local' }), 'local')
+testArchivistRoundTrip(new XyoStorageArchivist({ namespace: 'test', type: 'session' }), 'session')
+testArchivistRoundTrip(new XyoStorageArchivist({ namespace: 'test', type: 'page' }), 'page')
+
+testArchivistAll(new XyoStorageArchivist({ namespace: 'test', type: 'local' }), 'local')
+testArchivistAll(new XyoStorageArchivist({ namespace: 'test', type: 'session' }), 'session')
+testArchivistAll(new XyoStorageArchivist({ namespace: 'test', type: 'page' }), 'page')

package/src/XyoStorageArchivist.ts

@@ -0,0 +1,167 @@
+import { assertEx } from '@xylabs/sdk-js'
+import { XyoBoundWitness } from '@xyo-network/boundwitness'
+import { XyoPayload, XyoPayloadWrapper } from '@xyo-network/payload'
+import { PromisableArray } from '@xyo-network/promisable'
+import compact from 'lodash/compact'
+import store, { StoreBase } from 'store2'
+
+import { XyoArchivistConfig } from './Config'
+import { PartialArchivistConfig } from './PartialArchivistConfig'
+import {
+  XyoArchivistAllQuerySchema,
+  XyoArchivistClearQuerySchema,
+  XyoArchivistCommitQuerySchema,
+  XyoArchivistDeleteQuerySchema,
+  XyoArchivistFindQuerySchema,
+  XyoArchivistInsertQuery,
+  XyoArchivistInsertQuerySchema,
+} from './Queries'
+import { XyoArchivist } from './XyoArchivist'
+import { XyoPayloadFindFilter } from './XyoPayloadFindFilter'
+
+export type XyoStorageArchivistConfigSchema = 'network.xyo.module.config.archivist.storage'
+export const XyoStorageArchivistConfigSchema: XyoStorageArchivistConfigSchema = 'network.xyo.module.config.archivist.storage'
+
+export type XyoStorageArchivistConfig = XyoArchivistConfig<{
+  schema: XyoStorageArchivistConfigSchema
+  type?: 'local' | 'session' | 'page'
+  namespace?: string
+  maxEntries?: number
+  maxEntrySize?: number
+}>
+
+class StorageArchivistError extends Error {
+  constructor(action: string, error: Error['cause'], message?: string) {
+    super(`Storage Archivist [${action}] failed${message ? ` (${message})` : ''}`, { cause: error })
+  }
+}
+
+export class XyoStorageArchivist extends XyoArchivist<XyoStorageArchivistConfig> {
+  public get type() {
+    return this.config?.type ?? 'local'
+  }
+
+  public get namespace() {
+    return this.config?.namespace ?? 'xyoarch'
+  }
+
+  public get maxEntries() {
+    return this.config?.maxEntries ?? 1000
+  }
+
+  public get maxEntrySize() {
+    return this.config?.maxEntries ?? 16000
+  }
+
+  public override queries() {
+    return [
+      XyoArchivistAllQuerySchema,
+      XyoArchivistDeleteQuerySchema,
+      XyoArchivistClearQuerySchema,
+      XyoArchivistFindQuerySchema,
+      XyoArchivistCommitQuerySchema,
+      ...super.queries(),
+    ]
+  }
+
+  private storage: StoreBase
+
+  constructor(config?: PartialArchivistConfig<XyoStorageArchivistConfig>) {
+    super({ ...config, schema: XyoStorageArchivistConfigSchema })
+    this.storage = store[this.type].namespace(this.namespace)
+  }
+
+  public delete(hashes: string[]): PromisableArray<boolean> {
+    try {
+      return hashes.map((hash) => {
+        this.storage.remove(hash)
+        return true
+      })
+    } catch (ex) {
+      throw new StorageArchivistError('delete', ex, 'unexpected')
+    }
+  }
+
+  public override clear(): void | Promise<void> {
+    try {
+      this.storage.clear()
+    } catch (ex) {
+      throw new StorageArchivistError('clear', ex, 'unexpected')
+    }
+  }
+
+  public async get(hashes: string[]): Promise<(XyoPayload | null)[]> {
+    try {
+      return await Promise.all(
+        hashes.map(async (hash) => {
+          const value = this.storage.get(hash)
+          return value ?? (await this.getFromParents(hash)) ?? null
+        }),
+      )
+    } catch (ex) {
+      throw new StorageArchivistError('get', ex, 'unexpected')
+    }
+  }
+
+  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness> {
+    try {
+      const storedPayloads = payloads.map((payload) => {
+        const wrapper = new XyoPayloadWrapper(payload)
+        const hash = wrapper.hash
+        const value = JSON.stringify(wrapper.payload)
+        assertEx(value.length < this.maxEntrySize, `Payload too large [${wrapper.hash}, ${value.length}]`)
+        this.storage.set(hash, wrapper.payload)
+        return wrapper.payload
+      })
+      const boundwitness = this.bindPayloads(storedPayloads)
+      if (this.writeThrough) {
+        await this.writeToParents([boundwitness, ...storedPayloads])
+      }
+      return boundwitness
+    } catch (ex) {
+      throw new StorageArchivistError('insert', ex, 'unexpected')
+    }
+  }
+
+  public async find(filter: XyoPayloadFindFilter): Promise<XyoPayload[]> {
+    try {
+      const x = (await this.all()).filter((payload) => {
+        if (filter.schema && filter.schema !== payload.schema) {
+          return false
+        }
+        return true
+      })
+      return x
+    } catch (ex) {
+      throw new StorageArchivistError('find', ex, 'unexpected')
+    }
+  }
+
+  public all(): PromisableArray<XyoPayload> {
+    try {
+      return Object.entries(this.storage.getAll()).map(([, value]) => value)
+    } catch (ex) {
+      throw new StorageArchivistError('all', ex, 'unexpected')
+    }
+  }
+
+  public async commit(): Promise<XyoBoundWitness> {
+    try {
+      const payloads = await this.all()
+      assertEx(payloads.length > 0, 'Nothing to commit')
+      const block = this.bindPayloads(payloads)
+      await Promise.allSettled(
+        compact(
+          Object.values(this.parents?.commit ?? [])?.map(async (parent) => {
+            const query: XyoArchivistInsertQuery = { payloads: [block, ...payloads], schema: XyoArchivistInsertQuerySchema }
+            return await parent?.query(query)
+          }),
+        ),
+      )
+      await this.clear()
+      return block
+    } catch (ex) {
+      throw new StorageArchivistError('commit', ex, 'unexpected')
+    }
+  }
+}

package/src/index.ts

@@ -1,7 +1,10 @@
 export * from './Archivist'
 export * from './Config'
+export * from './PartialArchivistConfig'
 export * from './Queries'
 export * from './XyoArchivist'
 export * from './XyoArchivistWrapper'
+export * from './XyoCookieArchivist'
 export * from './XyoMemoryArchivist'
 export * from './XyoPayloadFindFilter'
+export * from './XyoStorageArchivist'

package/src/test.spec.test/index.ts

@@ -0,0 +1 @@
+export * from './testArchivist'

package/src/test.spec.test/testArchivist.ts

@@ -0,0 +1,48 @@
+/**
+ * @jest-environment jsdom
+ */
+
+import { delay } from '@xylabs/delay'
+import { XyoPayload, XyoPayloadWrapper } from '@xyo-network/payload'
+
+import { XyoArchivist } from '../XyoArchivist'
+import { XyoArchivistWrapper } from '../XyoArchivistWrapper'
+
+export const testArchivistRoundTrip = (archivist: XyoArchivist, name: string) => {
+  test(`XyoArchivist [${name}]`, async () => {
+    const idPayload: XyoPayload<{ salt: string }> = {
+      salt: 'test',
+      schema: 'network.xyo.id',
+    }
+    const payloadWrapper = new XyoPayloadWrapper(idPayload)
+    const archivistWrapper = new XyoArchivistWrapper(archivist)
+    const insertResult = await archivistWrapper.insert([idPayload])
+    expect(insertResult).toBeDefined()
+    expect(insertResult.payload_hashes.find((hash) => hash === payloadWrapper.hash)).toBeDefined()
+    const getResult = await archivistWrapper.get([payloadWrapper.hash])
+    expect(getResult).toBeDefined()
+    expect(getResult.length).toBe(1)
+    const gottenPayload = getResult[0]
+    if (gottenPayload) {
+      const gottenPayloadWrapper = new XyoPayloadWrapper(gottenPayload)
+      expect(gottenPayloadWrapper.hash).toBe(payloadWrapper.hash)
+    }
+  })
+}
+
+export const testArchivistAll = (archivist: XyoArchivist, name: string) => {
+  test(`XyoArchivist [${name}]`, async () => {
+    const idPayload = {
+      salt: 'test',
+      schema: 'network.xyo.id',
+    }
+    const archivistWrapper = new XyoArchivistWrapper(archivist)
+    for (let x = 0; x < 10; x++) {
+      await archivistWrapper.insert([{ ...idPayload, salt: Date.now().toString() } as XyoPayload<{ salt: string }>])
+      await delay(10)
+    }
+    const getResult = await archivistWrapper.all()
+    expect(getResult).toBeDefined()
+    expect(getResult.length).toBe(11)
+  })
+}