@xyo-network/archivist-abstract 2.72.9

package/dist/index.d.mts

@@ -0,0 +1,7 @@
+export { AbstractArchivist, ActionConfig, ArchivistParentInstances, InsertConfig } from './AbstractArchivist.mjs';
+import '@xyo-network/payload-model';
+import '@xyo-network/archivist-model';
+import '@xyo-network/boundwitness-builder';
+import '@xyo-network/boundwitness-model';
+import '@xyo-network/module';
+import '@xyo-network/promise';

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export { AbstractArchivist, ActionConfig, ArchivistParentInstances, InsertConfig } from './AbstractArchivist.js';
+import '@xyo-network/payload-model';
+import '@xyo-network/archivist-model';
+import '@xyo-network/boundwitness-builder';
+import '@xyo-network/boundwitness-model';
+import '@xyo-network/module';
+import '@xyo-network/promise';

package/dist/index.js

@@ -0,0 +1,23 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var src_exports = {};
+module.exports = __toCommonJS(src_exports);
+__reExport(src_exports, require("./AbstractArchivist"), module.exports);
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ...require("./AbstractArchivist")
+});
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["export * from './AbstractArchivist'\n"],"mappings":";;;;;;;;;;;;;;;AAAA;AAAA;AAAA,wBAAc,gCAAd;","names":[]}

package/dist/index.mjs

@@ -0,0 +1,2 @@
+export * from "./AbstractArchivist";
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["export * from './AbstractArchivist'\n"],"mappings":"AAAA,cAAc;","names":[]}

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "@xyo-network/archivist-abstract",
+  "author": {
+    "email": "support@xyo.network",
+    "name": "XYO Development Team",
+    "url": "https://xyo.network"
+  },
+  "bugs": {
+    "email": "support@xyo.network",
+    "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js/issues"
+  },
+  "dependencies": {
+    "@xylabs/assert": "^2.9.3",
+    "@xyo-network/archivist-model": "~2.72.9",
+    "@xyo-network/boundwitness-builder": "~2.72.9",
+    "@xyo-network/boundwitness-model": "~2.72.9",
+    "@xyo-network/core": "~2.72.9",
+    "@xyo-network/module": "~2.72.9",
+    "@xyo-network/payload-model": "~2.72.9",
+    "@xyo-network/payload-wrapper": "~2.72.9",
+    "@xyo-network/promise": "~2.72.9",
+    "lodash": "^4.17.21"
+  },
+  "devDependencies": {
+    "@xylabs/ts-scripts-yarn3": "^2.19.3",
+    "@xylabs/tsconfig": "^2.19.3",
+    "publint": "^0.2.2",
+    "tsup": "^7.2.0",
+    "typescript": "^5.2.2"
+  },
+  "description": "Primary SDK for using XYO Protocol 2.0",
+  "docs": "dist/docs.json",
+  "exports": {
+    ".": {
+      "require": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "import": {
+        "types": "./dist/index.d.mts",
+        "default": "./dist/index.mjs"
+      }
+    },
+    "./dist/docs.json": {
+      "default": "./dist/docs.json"
+    },
+    "./cjs": {
+      "default": "./dist/index.js"
+    },
+    "./docs": {
+      "default": "./dist/docs.json"
+    },
+    "./esm": {
+      "default": "./dist/index.mjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "main": "dist/index.js",
+  "module": "dist/index.mjs",
+  "homepage": "https://xyo.network",
+  "license": "LGPL-3.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "package-compile": "tsup && publint",
+    "package-recompile": "tsup && publint"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
+  },
+  "sideEffects": false,
+  "types": "dist/index.d.ts",
+  "version": "2.72.9"
+}

package/src/AbstractArchivist.ts

@@ -0,0 +1,350 @@
+import { assertEx } from '@xylabs/assert'
+import {
+  ArchivistAllQuerySchema,
+  ArchivistClearQuerySchema,
+  ArchivistCommitQuerySchema,
+  ArchivistDeleteQuery,
+  ArchivistDeleteQuerySchema,
+  ArchivistGetQuerySchema,
+  ArchivistInsertQuerySchema,
+  ArchivistInstance,
+  ArchivistModule,
+  ArchivistModuleEventData,
+  ArchivistParams,
+  ArchivistQuery,
+  ArchivistQueryBase,
+  asArchivistInstance,
+  isArchivistInstance,
+} from '@xyo-network/archivist-model'
+import { QueryBoundWitness, QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-builder'
+import { BoundWitness, BoundWitnessSchema } from '@xyo-network/boundwitness-model'
+import { PayloadHasher } from '@xyo-network/core'
+import { AbstractModuleInstance, duplicateModules, ModuleConfig, ModuleQueryHandlerResult } from '@xyo-network/module'
+import { Payload } from '@xyo-network/payload-model'
+import { PayloadWrapper } from '@xyo-network/payload-wrapper'
+import { Promisable, PromisableArray } from '@xyo-network/promise'
+import compact from 'lodash/compact'
+
+export interface ActionConfig {
+  emitEvents?: boolean
+}
+
+export interface InsertConfig extends ActionConfig {
+  writeToParents?: boolean
+}
+
+export interface ArchivistParentInstances {
+  commit?: Record<string, ArchivistInstance>
+  read?: Record<string, ArchivistInstance>
+  write?: Record<string, ArchivistInstance>
+}
+
+export abstract class AbstractArchivist<
+    TParams extends ArchivistParams = ArchivistParams,
+    TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,
+  >
+  extends AbstractModuleInstance<TParams, TEventData>
+  implements ArchivistModule<TParams>
+{
+  private _lastInsertedPayload: Payload | undefined
+  private _parents?: ArchivistParentInstances
+
+  override get queries(): string[] {
+    return [ArchivistGetQuerySchema, ...super.queries]
+  }
+
+  get requireAllParents() {
+    return this.config.requireAllParents ?? true
+  }
+
+  protected override get _queryAccountPaths(): Record<ArchivistQueryBase['schema'], string> {
+    return {
+      'network.xyo.query.archivist.all': '1/1',
+      'network.xyo.query.archivist.clear': '1/2',
+      'network.xyo.query.archivist.commit': '1/3',
+      'network.xyo.query.archivist.delete': '1/4',
+      'network.xyo.query.archivist.get': '1/5',
+      'network.xyo.query.archivist.insert': '1/6',
+    }
+  }
+
+  protected get storeParentReads() {
+    return !!this.config?.storeParentReads
+  }
+
+  all(): PromisableArray<Payload> {
+    this._noOverride('all')
+    return this.busy(async () => {
+      await this.started('throw')
+      return await this.allHandler()
+    })
+  }
+
+  clear(): Promisable<void> {
+    this._noOverride('clear')
+    return this.busy(async () => {
+      await this.started('throw')
+      return await this.clearHandler()
+    })
+  }
+
+  commit(): Promisable<BoundWitness[]> {
+    this._noOverride('commit')
+    return this.busy(async () => {
+      await this.started('throw')
+      return await this.commitHandler()
+    })
+  }
+
+  async delete(hashes: string[]): Promise<string[]> {
+    this._noOverride('delete')
+    return await this.busy(async () => {
+      await this.started('throw')
+      return await this.deleteWithConfig(hashes)
+    })
+  }
+
+  async get(hashes: string[]): Promise<Payload[]> {
+    this._noOverride('get')
+    return await this.busy(async () => {
+      await this.started('throw')
+      return await this.getWithConfig(hashes)
+    })
+  }
+
+  async insert(payloads: Payload[]): Promise<Payload[]> {
+    this._noOverride('insert')
+    return await this.busy(async () => {
+      await this.started('throw')
+      return await this.insertWithConfig(payloads)
+    })
+  }
+
+  protected allHandler(): PromisableArray<Payload> {
+    throw Error('Not implemented')
+  }
+
+  protected clearHandler(): Promisable<void> {
+    throw Error('Not implemented')
+  }
+
+  protected commitHandler(): Promisable<BoundWitness[]> {
+    throw Error('Not implemented')
+  }
+
+  protected deleteHandler(_hashes: string[]): PromisableArray<string> {
+    throw Error('Not implemented')
+  }
+
+  protected async deleteWithConfig(hashes: string[], config?: ActionConfig): Promise<string[]> {
+    const emitEvents = config?.emitEvents ?? true
+
+    const deletedHashes = await this.deleteHandler(hashes)
+
+    if (emitEvents) {
+      await this.emit('deleted', { hashes: deletedHashes, module: this })
+    }
+
+    return deletedHashes
+  }
+
+  protected async getFromParent(hashes: string[], archivist: ArchivistInstance): Promise<[Payload[], string[]]> {
+    const foundPairs = (
+      await Promise.all(
+        (await archivist.get(hashes)).map<Promise<[string, Payload]>>(async (payload) => [await PayloadHasher.hashAsync(payload), payload]),
+      )
+    ).filter(([hash]) => {
+      const askedFor = hashes.includes(hash)
+      if (!askedFor) {
+        console.warn(`Parent returned payload with hash not asked for: ${hash}`)
+        //throw Error(`Parent returned payload with hash not asked for: ${hash}`)
+      }
+      return askedFor
+    })
+
+    const foundHashes = foundPairs.map(([hash]) => hash)
+    const foundPayloads = foundPairs.map(([, payload]) => payload)
+
+    const notfound = hashes.filter((hash) => !foundHashes.includes(hash))
+    return [foundPayloads, notfound]
+  }
+
+  protected async getFromParents(hashes: string[]): Promise<[Payload[], string[]]> {
+    const parents = Object.values((await this.parents())?.read ?? {})
+    let remainingHashes = [...hashes]
+    let parentIndex = 0
+    let result: Payload[] = []
+
+    //intentionally doing this serially
+    while (parentIndex < parents.length && remainingHashes.length > 0) {
+      const [found, notfound] = await this.getFromParent(remainingHashes, parents[parentIndex])
+      result = [...result, ...found]
+      remainingHashes = notfound
+      parentIndex++
+    }
+    return [result, remainingHashes]
+  }
+
+  protected getHandler(_hashes: string[]): Promisable<Payload[]> {
+    throw Error('Not implemented')
+  }
+
+  protected async getWithConfig(hashes: string[], config?: InsertConfig): Promise<Payload[]> {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const emitEvents = config?.emitEvents ?? true
+    const map = await PayloadWrapper.toMap(await this.getHandler(hashes))
+
+    const { foundPayloads, notfoundHashes } = hashes.reduce<{ foundPayloads: Payload[]; notfoundHashes: string[] }>(
+      (prev, hash) => {
+        const found = map[hash]
+        if (found) {
+          //TODO: Find a better way to scrub meta data without scrubbing _signatures
+          if (found.schema === BoundWitnessSchema) {
+            prev.foundPayloads.push({ ...PayloadHasher.hashFields(found), ...{ _signatures: (found as BoundWitness)._signatures } })
+          } else {
+            prev.foundPayloads.push({ ...PayloadHasher.hashFields(found) })
+          }
+        } else {
+          prev.notfoundHashes.push(hash)
+        }
+        return prev
+      },
+      { foundPayloads: [], notfoundHashes: [] },
+    )
+
+    const [parentFoundPayloads] = await this.getFromParents(notfoundHashes)
+
+    if (this.storeParentReads) {
+      await this.insertWithConfig(parentFoundPayloads)
+    }
+    return [...foundPayloads, ...parentFoundPayloads]
+  }
+
+  protected head(): Promisable<Payload | undefined> {
+    return this._lastInsertedPayload
+  }
+
+  protected insertHandler(_payloads: Payload[]): Promise<Payload[]> {
+    throw Error('Not implemented')
+  }
+
+  protected async insertWithConfig(payloads: Payload[], config?: InsertConfig): Promise<Payload[]> {
+    const emitEvents = config?.emitEvents ?? true
+    const writeToParents = config?.writeToParents ?? true
+
+    const insertedPayloads = await this.insertHandler(payloads)
+
+    if (writeToParents) {
+      await this.writeToParents(insertedPayloads)
+    }
+    if (emitEvents) {
+      await this.emit('inserted', { module: this, payloads: insertedPayloads })
+    }
+
+    return insertedPayloads
+  }
+
+  protected async parents() {
+    this._parents = this._parents ?? {
+      commit: await this.resolveArchivists(this.config?.parents?.commit),
+      read: await this.resolveArchivists(this.config?.parents?.read),
+      write: await this.resolveArchivists(this.config?.parents?.write),
+    }
+    return assertEx(this._parents)
+  }
+
+  protected override async queryHandler<T extends QueryBoundWitness = QueryBoundWitness, TConfig extends ModuleConfig = ModuleConfig>(
+    query: T,
+    payloads?: Payload[],
+    queryConfig?: TConfig,
+  ): Promise<ModuleQueryHandlerResult> {
+    const wrappedQuery = QueryBoundWitnessWrapper.parseQuery<ArchivistQuery>(query, payloads)
+    const queryPayload = await wrappedQuery.getQuery()
+    assertEx(this.queryable(query, payloads, queryConfig))
+    const resultPayloads: Payload[] = []
+    if (this.config.storeQueries) {
+      await this.insertHandler([query])
+    }
+
+    switch (queryPayload.schema) {
+      case ArchivistAllQuerySchema:
+        resultPayloads.push(...(await this.allHandler()))
+        break
+      case ArchivistClearQuerySchema:
+        await this.clearHandler()
+        break
+      case ArchivistCommitQuerySchema:
+        resultPayloads.push(...(await this.commitHandler()))
+        break
+      case ArchivistDeleteQuerySchema: {
+        const resultPayload: ArchivistDeleteQuery = {
+          hashes: [...(await this.deleteWithConfig(queryPayload.hashes))],
+          schema: ArchivistDeleteQuerySchema,
+        }
+        resultPayloads.push(resultPayload)
+        break
+      }
+      case ArchivistGetQuerySchema:
+        if (queryPayload.hashes?.length) {
+          resultPayloads.push(...(await this.getWithConfig(queryPayload.hashes)))
+        } else {
+          const head = await this.head()
+          if (head) resultPayloads.push(head)
+        }
+        break
+      case ArchivistInsertQuerySchema: {
+        const payloads = await wrappedQuery.getPayloads()
+        assertEx(await wrappedQuery.getPayloads(), `Missing payloads: ${JSON.stringify(wrappedQuery.payload(), null, 2)}`)
+        const resolvedPayloads = await PayloadWrapper.filterExclude(payloads, await wrappedQuery.hashAsync())
+        assertEx(resolvedPayloads.length === payloads.length, `Could not find some passed hashes [${resolvedPayloads.length} != ${payloads.length}]`)
+        resultPayloads.push(...(await this.insertWithConfig(payloads)))
+        // NOTE: There isn't an exact equivalence between what we get and what we store. Once
+        // we move to returning only inserted Payloads(/hash) instead of a BoundWitness, we
+        // can grab the actual last one
+        this._lastInsertedPayload = resolvedPayloads[resolvedPayloads.length - 1]
+        break
+      }
+      default:
+        return await super.queryHandler(query, payloads)
+    }
+    return resultPayloads
+  }
+
+  protected async writeToParent(parent: ArchivistInstance, payloads: Payload[]) {
+    return await parent.insert(payloads)
+  }
+
+  protected async writeToParents(payloads: Payload[]): Promise<Payload[]> {
+    const parents = await this.parents()
+    this.logger?.log(parents.write?.length ?? 0)
+    return compact(
+      await Promise.all(
+        Object.values(parents.write ?? {}).map(async (parent) => {
+          return parent ? await this.writeToParent(parent, payloads) : undefined
+        }),
+      ),
+    ).flat()
+  }
+
+  private async resolveArchivists(archivists: string[] = []) {
+    const archivistModules = [...(await this.resolve({ address: archivists })), ...(await this.resolve({ name: archivists }))].filter(
+      duplicateModules,
+    )
+
+    assertEx(
+      !this.requireAllParents || archivistModules.length === archivists.length,
+      `Failed to find some archivists (set allRequired to false if ok): [${archivists.filter((archivist) =>
+        archivistModules.map((module) => !(module.address === archivist || module.config.name === archivist)),
+      )}]`,
+    )
+
+    return archivistModules.reduce<Record<string, ArchivistInstance>>((prev, module) => {
+      prev[module.address] = asArchivistInstance(module, () => {
+        isArchivistInstance(module, { log: console })
+        return `Unable to cast resolved module to an archivist: [${module.address}, ${module.config.name}, ${module.config.schema})}]`
+      })
+
+      return prev
+    }, {})
+  }
+}

package/src/index.ts

@@ -0,0 +1 @@
+export * from './AbstractArchivist'

package/tsup.config.ts

@@ -0,0 +1,14 @@
+import { defineConfig } from 'tsup'
+
+// eslint-disable-next-line import/no-default-export
+export default defineConfig({
+  bundle: false,
+  cjsInterop: true,
+  clean: true,
+  dts: true,
+  entry: ['src'],
+  format: ['cjs', 'esm'],
+  sourcemap: true,
+  splitting: false,
+  tsconfig: 'tsconfig.json',
+})

package/typedoc.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://typedoc.org/schema.json",
+  "entryPoints": ["src/index.ts"],
+  "tsconfig": "./tsconfig.typedoc.json"
+}