@xyo-network/archivist 2.31.4 → 2.32.0-rc.2

package/src/Config.ts

@@ -16,26 +16,3 @@ export type XyoArchivistConfig<TConfig extends XyoPayload = XyoPayload> = XyoMod
     writeThrough?: string
   } & TConfig
 >
-
-export class XyoArchivistConfigWrapper<C extends XyoArchivistConfig = XyoArchivistConfig> {
-  protected config?: C
-  constructor(config?: C) {
-    this.config = config
-  }
-
-  public get parents() {
-    return this.config?.parents
-  }
-
-  public get account() {
-    return this.config?.account
-  }
-
-  public get writeThrough() {
-    return this.config?.writeThrough
-  }
-
-  public get cacheParentReads() {
-    return this.config?.cacheParentReads
-  }
-}

package/src/Queries/All.ts

@@ -0,0 +1,8 @@
+import { XyoQuery } from '@xyo-network/module'
+
+export type XyoArchivistAllQuerySchema = 'network.xyo.query.archivist.all'
+export const XyoArchivistAllQuerySchema: XyoArchivistAllQuerySchema = 'network.xyo.query.archivist.all'
+
+export type XyoArchivistAllQuery = XyoQuery<{
+  schema: XyoArchivistAllQuerySchema
+}>

package/src/Queries/Clear.ts

@@ -0,0 +1,8 @@
+import { XyoQuery } from '@xyo-network/module'
+
+export type XyoArchivistClearQuerySchema = 'network.xyo.query.archivist.clear'
+export const XyoArchivistClearQuerySchema: XyoArchivistClearQuerySchema = 'network.xyo.query.archivist.clear'
+
+export type XyoArchivistClearQuery = XyoQuery<{
+  schema: XyoArchivistClearQuerySchema
+}>

package/src/Queries/Commit.ts

@@ -0,0 +1,8 @@
+import { XyoQuery } from '@xyo-network/module'
+
+export type XyoArchivistCommitQuerySchema = 'network.xyo.query.archivist.commit'
+export const XyoArchivistCommitQuerySchema: XyoArchivistCommitQuerySchema = 'network.xyo.query.archivist.commit'
+
+export type XyoArchivistCommitQuery = XyoQuery<{
+  schema: XyoArchivistCommitQuerySchema
+}>

package/src/Queries/Delete.ts

@@ -0,0 +1,9 @@
+import { XyoQuery } from '@xyo-network/module'
+
+export type XyoArchivistDeleteQuerySchema = 'network.xyo.query.archivist.delete'
+export const XyoArchivistDeleteQuerySchema: XyoArchivistDeleteQuerySchema = 'network.xyo.query.archivist.delete'
+
+export type XyoArchivistDeleteQuery = XyoQuery<{
+  schema: XyoArchivistDeleteQuerySchema
+  hashes: string[]
+}>

package/src/Queries/Find.ts

@@ -0,0 +1,11 @@
+import { XyoQuery } from '@xyo-network/module'
+
+import { XyoPayloadFindFilter } from '../XyoPayloadFindFilter'
+
+export type XyoArchivistFindQuerySchema = 'network.xyo.query.archivist.find'
+export const XyoArchivistFindQuerySchema: XyoArchivistFindQuerySchema = 'network.xyo.query.archivist.find'
+
+export type XyoArchivistFindQuery = XyoQuery<{
+  schema: XyoArchivistFindQuerySchema
+  filter: XyoPayloadFindFilter
+}>

package/src/Queries/Get.ts

@@ -0,0 +1,9 @@
+import { XyoQuery } from '@xyo-network/module'
+
+export type XyoArchivistGetQuerySchema = 'network.xyo.query.archivist.get'
+export const XyoArchivistGetQuerySchema: XyoArchivistGetQuerySchema = 'network.xyo.query.archivist.get'
+
+export type XyoArchivistGetQuery = XyoQuery<{
+  schema: XyoArchivistGetQuerySchema
+  hashes: string[]
+}>

package/src/Queries/Insert.ts

@@ -0,0 +1,10 @@
+import { XyoQuery } from '@xyo-network/module'
+import { XyoPayload } from '@xyo-network/payload'
+
+export type XyoArchivistInsertQuerySchema = 'network.xyo.query.archivist.insert'
+export const XyoArchivistInsertQuerySchema: XyoArchivistInsertQuerySchema = 'network.xyo.query.archivist.insert'
+
+export type XyoArchivistInsertQuery = XyoQuery<{
+  schema: XyoArchivistInsertQuerySchema
+  payloads: XyoPayload[]
+}>

package/src/Queries/Payload.ts

@@ -0,0 +1,19 @@
+import { XyoModuleQuery } from '@xyo-network/module'
+
+import { XyoArchivistAllQuery } from './All'
+import { XyoArchivistClearQuery } from './Clear'
+import { XyoArchivistCommitQuery } from './Commit'
+import { XyoArchivistDeleteQuery } from './Delete'
+import { XyoArchivistFindQuery } from './Find'
+import { XyoArchivistGetQuery } from './Get'
+import { XyoArchivistInsertQuery } from './Insert'
+
+export type XyoArchivistQuery =
+  | XyoModuleQuery
+  | XyoArchivistAllQuery
+  | XyoArchivistClearQuery
+  | XyoArchivistCommitQuery
+  | XyoArchivistDeleteQuery
+  | XyoArchivistFindQuery
+  | XyoArchivistGetQuery
+  | XyoArchivistInsertQuery

package/src/Queries/Schema.ts

@@ -0,0 +1,20 @@
+import { XyoModuleQuerySchema } from '@xyo-network/module'
+
+import { XyoArchivistAllQuerySchema } from './All'
+import { XyoArchivistClearQuerySchema } from './Clear'
+import { XyoArchivistCommitQuerySchema } from './Commit'
+import { XyoArchivistDeleteQuerySchema } from './Delete'
+import { XyoArchivistFindQuerySchema } from './Find'
+import { XyoArchivistGetQuerySchema } from './Get'
+import { XyoArchivistInsertQuerySchema } from './Insert'
+
+export type XyoArchivistQuerySchema<T extends string = never> =
+  | XyoArchivistAllQuerySchema
+  | XyoArchivistClearQuerySchema
+  | XyoArchivistCommitQuerySchema
+  | XyoArchivistDeleteQuerySchema
+  | XyoArchivistFindQuerySchema
+  | XyoArchivistGetQuerySchema
+  | XyoArchivistInsertQuerySchema
+  | XyoModuleQuerySchema
+  | T

package/src/XyoArchivist.ts

@@ -0,0 +1,147 @@
+import { assertEx } from '@xylabs/sdk-js'
+import { XyoBoundWitness } from '@xyo-network/boundwitness'
+import { Module, XyoModule, XyoModuleInitializeQuerySchema, XyoModuleQueryResult, XyoModuleShutdownQuerySchema } from '@xyo-network/module'
+import { XyoPayload, XyoPayloadWrapper } from '@xyo-network/payload'
+import { NullablePromisableArray, Promisable, PromisableArray } from '@xyo-network/promisable'
+import compact from 'lodash/compact'
+
+import { Archivist } from './Archivist'
+import { XyoArchivistConfig, XyoArchivistParents } from './Config'
+import {
+  XyoArchivistAllQuerySchema,
+  XyoArchivistClearQuerySchema,
+  XyoArchivistCommitQuerySchema,
+  XyoArchivistDeleteQuerySchema,
+  XyoArchivistFindQuerySchema,
+  XyoArchivistGetQuery,
+  XyoArchivistGetQuerySchema,
+  XyoArchivistInsertQuery,
+  XyoArchivistInsertQuerySchema,
+  XyoArchivistQuery,
+  XyoArchivistQuerySchema,
+} from './Queries'
+import { XyoPayloadFindFilter } from './XyoPayloadFindFilter'
+
+export abstract class XyoArchivist<TConfig extends XyoPayload = XyoPayload>
+  extends XyoModule<XyoArchivistConfig<TConfig>>
+  implements Archivist<XyoPayload, XyoPayload, XyoPayload, XyoPayload, XyoPayloadFindFilter>
+{
+  public override get queries(): XyoArchivistQuerySchema[] {
+    return [XyoModuleInitializeQuerySchema, XyoModuleShutdownQuerySchema, XyoArchivistGetQuerySchema, XyoArchivistInsertQuerySchema]
+  }
+
+  public get cacheParentReads() {
+    return !!this.config?.cacheParentReads
+  }
+
+  public get writeThrough() {
+    return !!this.config?.writeThrough
+  }
+
+  public all(): PromisableArray<XyoPayload> {
+    throw Error('Not implemented')
+  }
+
+  public clear(): Promisable<void> {
+    throw Error('Not implemented')
+  }
+
+  public commit(): Promisable<XyoBoundWitness> {
+    throw Error('Not implemented')
+  }
+
+  public delete(_hashes: string[]): PromisableArray<boolean> {
+    throw Error('Not implemented')
+  }
+
+  public find(_filter: XyoPayloadFindFilter): PromisableArray<XyoPayload> {
+    throw Error('Not implemented')
+  }
+
+  abstract get(hashes: string[]): NullablePromisableArray<XyoPayload>
+
+  abstract insert(item: XyoPayload[]): Promisable<XyoBoundWitness>
+
+  async query(query: XyoArchivistQuery): Promise<XyoModuleQueryResult> {
+    if (!this.queries.find((schema) => schema === query.schema)) {
+      console.error(`Undeclared Module Query: ${query.schema}`)
+    }
+
+    const payloads: (XyoPayload | null)[] = []
+    switch (query.schema) {
+      case XyoArchivistAllQuerySchema:
+        payloads.push(...(await this.all()))
+        break
+      case XyoArchivistClearQuerySchema:
+        await this.clear()
+        break
+      case XyoArchivistCommitQuerySchema:
+        payloads.push(await this.commit())
+        break
+      case XyoArchivistDeleteQuerySchema:
+        await this.delete(query.hashes)
+        break
+      case XyoArchivistFindQuerySchema:
+        payloads.push(...(await this.find(query.filter)))
+        break
+      case XyoArchivistGetQuerySchema:
+        payloads.push(...(await this.get(query.hashes)))
+        break
+      case XyoArchivistInsertQuerySchema:
+        payloads.push(await this.insert(query.payloads), ...query.payloads)
+        break
+    }
+    return [this.bindPayloads(payloads), payloads]
+  }
+
+  private resolveArchivists(archivists?: Record<string, Module | null | undefined>) {
+    const resolved: Record<string, Module | null | undefined> = {}
+    if (archivists) {
+      Object.entries(archivists).forEach(([key, value]) => {
+        resolved[key] = value ?? this.resolver?.(key) ?? null
+      })
+    }
+    return resolved
+  }
+
+  protected async getFromParents(hash: string) {
+    return compact(
+      await Promise.all(
+        Object.values(this.parents?.read ?? {}).map(async (parent) => {
+          const query: XyoArchivistGetQuery = { hashes: [hash], schema: XyoArchivistGetQuerySchema }
+          const [, payloads] = (await parent?.query(query)) ?? []
+          const wrapper = payloads?.[0] ? new XyoPayloadWrapper(payloads?.[0]) : undefined
+          if (wrapper && wrapper.hash !== hash) {
+            console.warn(`Parent [${parent?.address}] returned payload with invalid hash [${hash} != ${wrapper.hash}]`)
+            return null
+          }
+          return wrapper?.payload
+        }),
+      ),
+    )[0]
+  }
+
+  protected async writeToParent(parent: Module, payloads: XyoPayload[]) {
+    const query: XyoArchivistInsertQuery = { payloads, schema: XyoArchivistInsertQuerySchema }
+    const [, writtenPayloads] = (await parent?.query(query)) ?? []
+    return writtenPayloads
+  }
+
+  protected async writeToParents(payloads: XyoPayload[]) {
+    return await Promise.all(
+      Object.values(this.parents?.write ?? {}).map(async (parent) => {
+        return parent ? await this.writeToParent(parent, payloads) : undefined
+      }),
+    )
+  }
+
+  private _parents?: XyoArchivistParents
+  get parents() {
+    this._parents = this._parents ?? {
+      commit: this.resolveArchivists(this.config?.parents?.commit),
+      read: this.resolveArchivists(this.config?.parents?.read),
+      write: this.resolveArchivists(this.config?.parents?.write),
+    }
+    return assertEx(this._parents)
+  }
+}

package/src/XyoArchivistWrapper.ts

@@ -1,68 +1,59 @@
-import { XyoModule } from '@xyo-network/module'
+import { XyoBoundWitness } from '@xyo-network/boundwitness'
+import { XyoModuleWrapper } from '@xyo-network/module'
 import { XyoPayload } from '@xyo-network/payload'
 
 import { Archivist } from './Archivist'
 import {
-  XyoArchivistAllQueryPayload,
-  XyoArchivistAllQueryPayloadSchema,
-  XyoArchivistClearQueryPayload,
-  XyoArchivistClearQueryPayloadSchema,
-  XyoArchivistCommitQueryPayload,
-  XyoArchivistCommitQueryPayloadSchema,
-  XyoArchivistDeleteQueryPayload,
-  XyoArchivistDeleteQueryPayloadSchema,
-  XyoArchivistFindQueryPayload,
-  XyoArchivistFindQueryPayloadSchema,
-  XyoArchivistGetQueryPayload,
-  XyoArchivistGetQueryPayloadSchema,
-  XyoArchivistInsertQueryPayload,
-  XyoArchivistInsertQueryPayloadSchema,
-} from './Query'
+  XyoArchivistAllQuery,
+  XyoArchivistAllQuerySchema,
+  XyoArchivistClearQuery,
+  XyoArchivistClearQuerySchema,
+  XyoArchivistCommitQuery,
+  XyoArchivistCommitQuerySchema,
+  XyoArchivistDeleteQuery,
+  XyoArchivistDeleteQuerySchema,
+  XyoArchivistFindQuery,
+  XyoArchivistFindQuerySchema,
+  XyoArchivistGetQuery,
+  XyoArchivistGetQuerySchema,
+  XyoArchivistInsertQuery,
+  XyoArchivistInsertQuerySchema,
+} from './Queries'
 import { XyoPayloadFindFilter } from './XyoPayloadFindFilter'
 
-export class XyoArchivistWrapper implements Archivist<XyoPayload | null, XyoPayload | null, XyoPayload, XyoPayload | null, XyoPayloadFindFilter> {
-  protected module: XyoModule
-
-  constructor(module: XyoModule) {
-    this.module = module
-  }
-
-  public get queries() {
-    return this.module.queries
-  }
-
+export class XyoArchivistWrapper extends XyoModuleWrapper implements Archivist {
   public async delete(hashes: string[]) {
-    const query: XyoArchivistDeleteQueryPayload = { hashes, schema: XyoArchivistDeleteQueryPayloadSchema }
+    const query: XyoArchivistDeleteQuery = { hashes, schema: XyoArchivistDeleteQuerySchema }
     return (await this.module.query(query))[0].payload_hashes.map(() => true)
   }
 
   public async clear(): Promise<void> {
-    const query: XyoArchivistClearQueryPayload = { schema: XyoArchivistClearQueryPayloadSchema }
+    const query: XyoArchivistClearQuery = { schema: XyoArchivistClearQuerySchema }
     await this.module.query(query)
   }
 
   public async get(hashes: string[]): Promise<(XyoPayload | null)[]> {
-    const query: XyoArchivistGetQueryPayload = { hashes, schema: XyoArchivistGetQueryPayloadSchema }
+    const query: XyoArchivistGetQuery = { hashes, schema: XyoArchivistGetQuerySchema }
     return (await this.module.query(query))[1]
   }
 
-  public async insert(payloads: XyoPayload[]): Promise<(XyoPayload | null)[]> {
-    const query: XyoArchivistInsertQueryPayload = { payloads, schema: XyoArchivistInsertQueryPayloadSchema }
-    return (await this.module.query(query))[1]
+  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness> {
+    const query: XyoArchivistInsertQuery = { payloads, schema: XyoArchivistInsertQuerySchema }
+    return (await this.module.query(query))[0]
   }
 
   public async find(filter: XyoPayloadFindFilter): Promise<(XyoPayload | null)[]> {
-    const query: XyoArchivistFindQueryPayload = { filter, schema: XyoArchivistFindQueryPayloadSchema }
+    const query: XyoArchivistFindQuery = { filter, schema: XyoArchivistFindQuerySchema }
     return (await this.module.query(query))[1]
   }
 
   public async all(): Promise<(XyoPayload | null)[]> {
-    const query: XyoArchivistAllQueryPayload = { schema: XyoArchivistAllQueryPayloadSchema }
+    const query: XyoArchivistAllQuery = { schema: XyoArchivistAllQuerySchema }
     return (await this.module.query(query))[1]
   }
 
-  public async commit(): Promise<(XyoPayload | null)[]> {
-    const query: XyoArchivistCommitQueryPayload = { schema: XyoArchivistCommitQueryPayloadSchema }
-    return (await this.module.query(query))[1]
+  public async commit(): Promise<XyoBoundWitness> {
+    const query: XyoArchivistCommitQuery = { schema: XyoArchivistCommitQuerySchema }
+    return (await this.module.query(query))[0]
   }
 }

package/src/XyoCookieArchivist.ts

@@ -0,0 +1,176 @@
+import { assertEx } from '@xylabs/sdk-js'
+import { XyoBoundWitness } from '@xyo-network/boundwitness'
+import { XyoPayload, XyoPayloadWrapper } from '@xyo-network/payload'
+import { PromisableArray } from '@xyo-network/promisable'
+import Cookies from 'js-cookie'
+import compact from 'lodash/compact'
+
+import { XyoArchivistConfig } from './Config'
+import { PartialArchivistConfig } from './PartialArchivistConfig'
+import {
+  XyoArchivistAllQuerySchema,
+  XyoArchivistClearQuerySchema,
+  XyoArchivistCommitQuerySchema,
+  XyoArchivistDeleteQuerySchema,
+  XyoArchivistFindQuerySchema,
+  XyoArchivistInsertQuery,
+  XyoArchivistInsertQuerySchema,
+} from './Queries'
+import { XyoArchivist } from './XyoArchivist'
+import { XyoPayloadFindFilter } from './XyoPayloadFindFilter'
+
+export type XyoCookieArchivistConfigSchema = 'network.xyo.module.config.archivist.cookie'
+export const XyoCookieArchivistConfigSchema: XyoCookieArchivistConfigSchema = 'network.xyo.module.config.archivist.cookie'
+
+export type XyoCookieArchivistConfig = XyoArchivistConfig<{
+  schema: XyoCookieArchivistConfigSchema
+  domain?: string
+  maxEntries?: number
+  maxEntrySize?: number
+  namespace?: string
+}>
+
+class CookieArchivistError extends Error {
+  constructor(action: string, error: Error['cause'], message?: string) {
+    super(`Cookie Archivist [${action}] failed${message ? ` (${message})` : ''}`, { cause: error })
+  }
+}
+
+export class XyoCookieArchivist extends XyoArchivist<XyoCookieArchivistConfig> {
+  public get domain() {
+    return this.config?.domain
+  }
+
+  public get namespace() {
+    return this.config?.namespace ?? 'xyoarch'
+  }
+
+  public get maxEntries() {
+    //all browsers support at least 60 cookies
+    return this.config?.maxEntries ?? 60
+  }
+
+  public get maxEntrySize() {
+    //all browsers support at least 4000 length per cookie
+    return this.config?.maxEntrySize ?? 4000
+  }
+
+  public override get queries() {
+    return [
+      ...super.queries,
+      XyoArchivistAllQuerySchema,
+      XyoArchivistDeleteQuerySchema,
+      XyoArchivistClearQuerySchema,
+      XyoArchivistFindQuerySchema,
+      XyoArchivistCommitQuerySchema,
+    ]
+  }
+
+  constructor(config?: PartialArchivistConfig<XyoCookieArchivistConfig>) {
+    super({ ...config, schema: XyoCookieArchivistConfigSchema })
+  }
+
+  private keyFromHash(hash: string) {
+    return `${this.namespace}-${hash}`
+  }
+
+  public delete(hashes: string[]): PromisableArray<boolean> {
+    try {
+      return hashes.map((hash) => {
+        Cookies.remove(this.keyFromHash(hash))
+        return true
+      })
+    } catch (ex) {
+      throw new CookieArchivistError('delete', ex, 'unexpected')
+    }
+  }
+
+  public override clear(): void | Promise<void> {
+    try {
+      Object.entries(Cookies.get()).map(([key]) => {
+        if (key.startsWith(`${this.namespace}-`)) {
+          Cookies.remove(key)
+        }
+      })
+    } catch (ex) {
+      throw new CookieArchivistError('clear', ex, 'unexpected')
+    }
+  }
+
+  public async get(hashes: string[]): Promise<(XyoPayload | null)[]> {
+    try {
+      return await Promise.all(
+        hashes.map(async (hash) => {
+          const cookieString = Cookies.get(this.keyFromHash(hash))
+          return cookieString ? JSON.parse(cookieString) : (await this.getFromParents(hash)) ?? null
+        }),
+      )
+    } catch (ex) {
+      throw new CookieArchivistError('get', ex, 'unexpected')
+    }
+  }
+
+  public async insert(payloads: XyoPayload[]): Promise<XyoBoundWitness> {
+    try {
+      const storedPayloads = payloads.map((payload) => {
+        const wrapper = new XyoPayloadWrapper(payload)
+        const key = this.keyFromHash(wrapper.hash)
+        const value = JSON.stringify(wrapper.payload)
+        assertEx(value.length < this.maxEntrySize, `Payload too large [${wrapper.hash}, ${value.length}]`)
+        Cookies.set(key, JSON.stringify(wrapper.payload))
+        return wrapper.payload
+      })
+      const boundwitness = this.bindPayloads(storedPayloads)
+      if (this.writeThrough) {
+        await this.writeToParents([boundwitness, ...storedPayloads])
+      }
+      return boundwitness
+    } catch (ex) {
+      throw new CookieArchivistError('insert', ex, 'unexpected')
+    }
+  }
+
+  public async find(filter: XyoPayloadFindFilter): Promise<XyoPayload[]> {
+    try {
+      const x = (await this.all()).filter((payload) => {
+        if (filter.schema && filter.schema !== payload.schema) {
+          return false
+        }
+        return true
+      })
+      return x
+    } catch (ex) {
+      throw new CookieArchivistError('find', ex, 'unexpected')
+    }
+  }
+
+  public all(): PromisableArray<XyoPayload> {
+    try {
+      return Object.entries(Cookies.get())
+        .filter(([key]) => key.startsWith(`${this.namespace}-`))
+        .map(([, value]) => JSON.parse(value))
+    } catch (ex) {
+      throw new CookieArchivistError('all', ex, 'unexpected')
+    }
+  }
+
+  public async commit(): Promise<XyoBoundWitness> {
+    try {
+      const payloads = await this.all()
+      assertEx(payloads.length > 0, 'Nothing to commit')
+      const block = this.bindPayloads(payloads)
+      await Promise.allSettled(
+        compact(
+          Object.values(this.parents?.commit ?? [])?.map(async (parent) => {
+            const query: XyoArchivistInsertQuery = { payloads: [block, ...payloads], schema: XyoArchivistInsertQuerySchema }
+            return await parent?.query(query)
+          }),
+        ),
+      )
+      await this.clear()
+      return block
+    } catch (ex) {
+      throw new CookieArchivistError('commit', ex, 'unexpected')
+    }
+  }
+}