@xyo-network/archivist-abstract 2.90.1 → 2.90.3

package/dist/node/index.js

@@ -5,8 +5,7 @@ var __name = (target, value) => __defProp(target, "name", { value, configurable:
 import { assertEx } from "@xylabs/assert";
 import { compact } from "@xylabs/lodash";
 import { ArchivistAllQuerySchema, ArchivistClearQuerySchema, ArchivistCommitQuerySchema, ArchivistDeleteQuerySchema, ArchivistGetQuerySchema, ArchivistInsertQuerySchema, asArchivistInstance, isArchivistInstance } from "@xyo-network/archivist-model";
-import { QueryBoundWitnessWrapper } from "@xyo-network/boundwitness-builder";
-import { PayloadHasher } from "@xyo-network/hash";
+import { QueryBoundWitnessWrapper } from "@xyo-network/boundwitness-wrapper";
 import { AbstractModuleInstance } from "@xyo-network/module-abstract";
 import { duplicateModules } from "@xyo-network/module-model";
 import { PayloadBuilder } from "@xyo-network/payload-builder";
@@ -75,7 +74,7 @@ var _AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
     this._noOverride("insert");
     return await this.busy(async () => {
       await this.started("throw");
-      return await this.insertWithConfig(payloads);
+      return await this.insertWithConfig(await Promise.all(payloads.map((payload) => PayloadBuilder.build(payload, true))));
     });
   }
   allHandler() {
@@ -152,7 +151,7 @@ var _AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
     for (const hash of hashes) {
       const found = map[hash] ?? dataMap[hash];
       if (found) {
-        foundPayloads.push(PayloadHasher.jsonPayload(found));
+        foundPayloads.push(await PayloadBuilder.build(found));
       } else {
         notfoundHashes.push(hash);
       }
@@ -172,6 +171,16 @@ var _AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
   insertHandler(_payloads) {
     throw new Error("Not implemented");
   }
+  async insertQueryHandler(query, payloads) {
+    assertEx(payloads, () => `Missing payloads: ${JSON.stringify(query.payload, null, 2)}`);
+    const resolvedPayloads = await PayloadBuilder.filterIncludeByDataHash(payloads, query.payloadHashes);
+    assertEx(resolvedPayloads.length === query.payloadHashes.length, () => `Could not find some passed hashes [${resolvedPayloads.length} != ${query.payloadHashes.length}]`);
+    const queryPayload = await query.getQuery();
+    const payloadsWithoutQuery = await PayloadBuilder.filterExclude(resolvedPayloads, await PayloadBuilder.dataHash(queryPayload));
+    const result = await this.insertWithConfig(payloadsWithoutQuery);
+    this._lastInsertedPayload = resolvedPayloads.at(-1);
+    return result;
+  }
   async insertWithConfig(payloads, config) {
     const emitEvents = (config == null ? void 0 : config.emitEvents) ?? true;
     const writeToParents = (config == null ? void 0 : config.writeToParents) ?? true;
@@ -202,11 +211,6 @@ var _AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
     const queryPayload = await wrappedQuery.getQuery();
     assertEx(await this.queryable(query, payloads, queryConfig));
     const resultPayloads = [];
-    if (this.config.storeQueries) {
-      await this.insertHandler([
-        query
-      ]);
-    }
     switch (queryPayload.schema) {
       case ArchivistAllQuerySchema: {
         resultPayloads.push(...await this.allHandler());
@@ -241,18 +245,24 @@ var _AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
         break;
       }
       case ArchivistInsertQuerySchema: {
-        assertEx(payloads, () => `Missing payloads: ${JSON.stringify(wrappedQuery.jsonPayload(), null, 2)}`);
-        const resolvedPayloads = await PayloadBuilder.filterIncludeByDataHash(payloads, query.payload_hashes);
-        assertEx(resolvedPayloads.length === query.payload_hashes.length, () => `Could not find some passed hashes [${resolvedPayloads.length} != ${query.payload_hashes.length}]`);
-        const payloadsWithoutQuery = await PayloadBuilder.filterExclude(resolvedPayloads, await PayloadBuilder.dataHash(queryPayload));
-        resultPayloads.push(...await this.insertWithConfig(payloadsWithoutQuery));
-        this._lastInsertedPayload = resolvedPayloads.at(-1);
+        resultPayloads.push(...await this.insertQueryHandler(wrappedQuery, payloads));
         break;
       }
       default: {
-        return await super.queryHandler(query, payloads);
+        const result = await super.queryHandler(query, payloads);
+        if (this.config.storeQueries) {
+          await this.insertHandler([
+            query
+          ]);
+        }
+        return result;
       }
     }
+    if (this.config.storeQueries) {
+      await this.insertHandler([
+        query
+      ]);
+    }
     return resultPayloads;
   }
   async writeToParent(parent, payloads) {

package/dist/node/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/AbstractArchivist.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { compact } from '@xylabs/lodash'\nimport { Promisable, PromisableArray } from '@xylabs/promise'\nimport {\n  ArchivistAllQuerySchema,\n  ArchivistClearQuerySchema,\n  ArchivistCommitQuerySchema,\n  ArchivistDeleteQuery,\n  ArchivistDeleteQuerySchema,\n  ArchivistGetQuerySchema,\n  ArchivistInsertQuerySchema,\n  ArchivistInstance,\n  ArchivistModuleEventData,\n  ArchivistParams,\n  ArchivistQuery,\n  ArchivistQueryBase,\n  asArchivistInstance,\n  isArchivistInstance,\n} from '@xyo-network/archivist-model'\nimport { QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-builder'\nimport { BoundWitness, QueryBoundWitness } from '@xyo-network/boundwitness-model'\nimport { PayloadHasher } from '@xyo-network/hash'\nimport { AbstractModuleInstance } from '@xyo-network/module-abstract'\nimport { duplicateModules, ModuleConfig, ModuleQueryHandlerResult } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, WithMeta } from '@xyo-network/payload-model'\n\nexport interface ActionConfig {\n  emitEvents?: boolean\n}\n\nexport interface InsertConfig extends ActionConfig {\n  writeToParents?: boolean\n}\n\nexport interface ArchivistParentInstances {\n  commit?: Record<string, ArchivistInstance>\n  read?: Record<string, ArchivistInstance>\n  write?: Record<string, ArchivistInstance>\n}\n\nexport abstract class AbstractArchivist<\n    TParams extends ArchivistParams = ArchivistParams,\n    TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n  >\n  extends AbstractModuleInstance<TParams, TEventData>\n  implements ArchivistInstance<TParams>\n{\n  private _lastInsertedPayload: Payload | undefined\n  private _parents?: ArchivistParentInstances\n\n  override get queries(): string[] {\n    return [ArchivistGetQuerySchema, ...super.queries]\n  }\n\n  get requireAllParents() {\n    return this.config.requireAllParents ?? false\n  }\n\n  protected override get _queryAccountPaths(): Record<ArchivistQueryBase['schema'], string> {\n    return {\n      'network.xyo.query.archivist.all': '1/1',\n      'network.xyo.query.archivist.clear': '1/2',\n      'network.xyo.query.archivist.commit': '1/3',\n      'network.xyo.query.archivist.delete': '1/4',\n      'network.xyo.query.archivist.get': '1/5',\n      'network.xyo.query.archivist.insert': '1/6',\n    }\n  }\n\n  protected get storeParentReads() {\n    return !!this.config?.storeParentReads\n  }\n\n  all(): PromisableArray<Payload> {\n    this._noOverride('all')\n    return this.busy(async () => {\n      await this.started('throw')\n      return await this.allHandler()\n    })\n  }\n\n  clear(): Promisable<void> {\n    this._noOverride('clear')\n    return this.busy(async () => {\n      await this.started('throw')\n      return await this.clearHandler()\n    })\n  }\n\n  commit(): Promisable<BoundWitness[]> {\n    this._noOverride('commit')\n    return this.busy(async () => {\n      await this.started('throw')\n      return await this.commitHandler()\n    })\n  }\n\n  async delete(hashes: string[]): Promise<string[]> {\n    this._noOverride('delete')\n    return await this.busy(async () => {\n      await this.started('throw')\n      return await this.deleteWithConfig(hashes)\n    })\n  }\n\n  async get(hashes: string[]): Promise<Payload[]> {\n    this._noOverride('get')\n    return await this.busy(async () => {\n      await this.started('throw')\n      return await this.getWithConfig(hashes)\n    })\n  }\n\n  async insert(payloads: Payload[]): Promise<Payload[]> {\n    this._noOverride('insert')\n    return await this.busy(async () => {\n      await this.started('throw')\n      return await this.insertWithConfig(payloads)\n    })\n  }\n\n  protected allHandler(): PromisableArray<Payload> {\n    throw new Error('Not implemented')\n  }\n\n  protected clearHandler(): Promisable<void> {\n    throw new Error('Not implemented')\n  }\n\n  protected commitHandler(): Promisable<BoundWitness[]> {\n    throw new Error('Not implemented')\n  }\n\n  protected deleteHandler(_hashes: string[]): PromisableArray<string> {\n    throw new Error('Not implemented')\n  }\n\n  protected async deleteWithConfig(hashes: string[], config?: ActionConfig): Promise<string[]> {\n    const emitEvents = config?.emitEvents ?? true\n\n    const deletedHashes = await this.deleteHandler(hashes)\n\n    if (emitEvents) {\n      await this.emit('deleted', { hashes: deletedHashes, module: this })\n    }\n\n    return deletedHashes\n  }\n\n  protected async getFromParent(hashes: string[], archivist: ArchivistInstance): Promise<[Payload[], string[]]> {\n    const foundPairs = (await PayloadBuilder.dataHashPairs(await archivist.get(hashes))).filter(([, hash]) => {\n      const askedFor = hashes.includes(hash)\n      if (!askedFor) {\n        console.warn(`Parent returned payload with hash not asked for: ${hash}`)\n        //throw Error(`Parent returned payload with hash not asked for: ${hash}`)\n      }\n      return askedFor\n    })\n\n    const foundHashes = new Set(foundPairs.map(([, hash]) => hash))\n    const foundPayloads = foundPairs.map(([payload]) => payload)\n\n    const notfound = hashes.filter((hash) => !foundHashes.has(hash))\n    return [foundPayloads, notfound]\n  }\n\n  protected async getFromParents(hashes: string[]): Promise<[Payload[], string[]]> {\n    const parents = Object.values((await this.parents())?.read ?? {})\n    let remainingHashes = [...hashes]\n    let parentIndex = 0\n    let result: Payload[] = []\n\n    //intentionally doing this serially\n    while (parentIndex < parents.length && remainingHashes.length > 0) {\n      const [found, notfound] = await this.getFromParent(remainingHashes, parents[parentIndex])\n      result = [...result, ...found]\n      remainingHashes = notfound\n      parentIndex++\n    }\n    return [result, remainingHashes]\n  }\n\n  protected getHandler(_hashes: string[]): Promisable<Payload[]> {\n    throw new Error('Not implemented')\n  }\n\n  protected async getWithConfig(hashes: string[], config?: InsertConfig): Promise<Payload[]> {\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const emitEvents = config?.emitEvents ?? true\n    const gotten = await this.getHandler(hashes)\n    const map = await PayloadBuilder.toHashMap(gotten)\n    const dataMap = await PayloadBuilder.toDataHashMap(gotten)\n\n    const foundPayloads: WithMeta<Payload>[] = []\n    const notfoundHashes: string[] = []\n    for (const hash of hashes) {\n      const found = map[hash] ?? dataMap[hash]\n      if (found) {\n        foundPayloads.push(PayloadHasher.jsonPayload(found) as WithMeta<Payload>)\n      } else {\n        notfoundHashes.push(hash)\n      }\n    }\n\n    const [parentFoundPayloads] = await this.getFromParents(notfoundHashes)\n\n    if (this.storeParentReads) {\n      await this.insertWithConfig(parentFoundPayloads)\n    }\n    return [...foundPayloads, ...parentFoundPayloads]\n  }\n\n  protected head(): Promisable<Payload | undefined> {\n    return this._lastInsertedPayload\n  }\n\n  protected insertHandler(_payloads: Payload[]): Promise<Payload[]> {\n    throw new Error('Not implemented')\n  }\n\n  protected async insertWithConfig(payloads: Payload[], config?: InsertConfig): Promise<Payload[]> {\n    const emitEvents = config?.emitEvents ?? true\n    const writeToParents = config?.writeToParents ?? true\n\n    const insertedPayloads = await this.insertHandler(payloads)\n\n    if (writeToParents) {\n      await this.writeToParents(insertedPayloads)\n    }\n    if (emitEvents) {\n      await this.emit('inserted', { module: this, payloads: insertedPayloads })\n    }\n\n    return insertedPayloads\n  }\n\n  protected async parents() {\n    this._parents = this._parents ?? {\n      commit: await this.resolveArchivists(this.config?.parents?.commit),\n      read: await this.resolveArchivists(this.config?.parents?.read),\n      write: await this.resolveArchivists(this.config?.parents?.write),\n    }\n    return assertEx(this._parents)\n  }\n\n  protected override async queryHandler<T extends QueryBoundWitness = QueryBoundWitness, TConfig extends ModuleConfig = ModuleConfig>(\n    query: T,\n    payloads: Payload[],\n    queryConfig?: TConfig,\n  ): Promise<ModuleQueryHandlerResult> {\n    const wrappedQuery = await QueryBoundWitnessWrapper.parseQuery<ArchivistQuery>(query, payloads)\n    const queryPayload = await wrappedQuery.getQuery()\n    assertEx(await this.queryable(query, payloads, queryConfig))\n    const resultPayloads: Payload[] = []\n    if (this.config.storeQueries) {\n      await this.insertHandler([query])\n    }\n\n    switch (queryPayload.schema) {\n      case ArchivistAllQuerySchema: {\n        resultPayloads.push(...(await this.allHandler()))\n        break\n      }\n      case ArchivistClearQuerySchema: {\n        await this.clearHandler()\n        break\n      }\n      case ArchivistCommitQuerySchema: {\n        resultPayloads.push(...(await this.commitHandler()))\n        break\n      }\n      case ArchivistDeleteQuerySchema: {\n        const resultPayload: ArchivistDeleteQuery = {\n          hashes: [...(await this.deleteWithConfig(queryPayload.hashes))],\n          schema: ArchivistDeleteQuerySchema,\n        }\n        resultPayloads.push(resultPayload)\n        break\n      }\n      case ArchivistGetQuerySchema: {\n        if (queryPayload.hashes?.length) {\n          resultPayloads.push(...(await this.getWithConfig(queryPayload.hashes)))\n        } else {\n          const head = await this.head()\n          if (head) resultPayloads.push(head)\n        }\n        break\n      }\n      case ArchivistInsertQuerySchema: {\n        assertEx(payloads, () => `Missing payloads: ${JSON.stringify(wrappedQuery.jsonPayload(), null, 2)}`)\n        const resolvedPayloads = await PayloadBuilder.filterIncludeByDataHash(payloads, query.payload_hashes)\n        assertEx(\n          resolvedPayloads.length === query.payload_hashes.length,\n          () => `Could not find some passed hashes [${resolvedPayloads.length} != ${query.payload_hashes.length}]`,\n        )\n        const payloadsWithoutQuery = await PayloadBuilder.filterExclude(resolvedPayloads, await PayloadBuilder.dataHash(queryPayload))\n        resultPayloads.push(...(await this.insertWithConfig(payloadsWithoutQuery)))\n        // NOTE: There isn't an exact equivalence between what we get and what we store. Once\n        // we move to returning only inserted Payloads(/hash) instead of a BoundWitness, we\n        // can grab the actual last one\n        this._lastInsertedPayload = resolvedPayloads.at(-1)\n        break\n      }\n      default: {\n        return await super.queryHandler(query, payloads)\n      }\n    }\n    return resultPayloads\n  }\n\n  protected async writeToParent(parent: ArchivistInstance, payloads: Payload[]) {\n    return await parent.insert(payloads)\n  }\n\n  protected async writeToParents(payloads: Payload[]): Promise<Payload[]> {\n    const parents = await this.parents()\n    return compact(\n      await Promise.all(\n        Object.values(parents.write ?? {}).map(async (parent) => {\n          return parent ? await this.writeToParent(parent, payloads) : undefined\n        }),\n      ),\n    ).flat()\n  }\n\n  private async resolveArchivists(archivists: string[] = []) {\n    const archivistModules = [...(await this.resolve({ address: archivists })), ...(await this.resolve({ name: archivists }))].filter(\n      duplicateModules,\n    )\n\n    assertEx(\n      !this.requireAllParents || archivistModules.length === archivists.length,\n      () =>\n        `Failed to find some archivists (set allRequired to false if ok): [${archivists.filter((archivist) =>\n          archivistModules.map((module) => !(module.address === archivist || module.config.name === archivist)),\n        )}]`,\n    )\n\n    // eslint-disable-next-line unicorn/no-array-reduce\n    return archivistModules.reduce<Record<string, ArchivistInstance>>((prev, module) => {\n      prev[module.address] = asArchivistInstance(module, () => {\n        isArchivistInstance(module, { log: console })\n        return `Unable to cast resolved module to an archivist: [${module.address}, ${module.config.name}, ${module.config.schema})}]`\n      })\n\n      return prev\n    }, {})\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,eAAe;AAExB,SACEC,yBACAC,2BACAC,4BAEAC,4BACAC,yBACAC,4BAMAC,qBACAC,2BACK;AACP,SAASC,gCAAgC;AAEzC,SAASC,qBAAqB;AAC9B,SAASC,8BAA8B;AACvC,SAASC,wBAAgE;AACzE,SAASC,sBAAsB;AAiBxB,IAAeC,qBAAf,MAAeA,2BAIZC,uBAAAA;EAGAC;EACAC;EAER,IAAaC,UAAoB;AAC/B,WAAO;MAACC;SAA4B,MAAMD;;EAC5C;EAEA,IAAIE,oBAAoB;AACtB,WAAO,KAAKC,OAAOD,qBAAqB;EAC1C;EAEA,IAAuBE,qBAAmE;AACxF,WAAO;MACL,mCAAmC;MACnC,qCAAqC;MACrC,sCAAsC;MACtC,sCAAsC;MACtC,mCAAmC;MACnC,sCAAsC;IACxC;EACF;EAEA,IAAcC,mBAAmB;AAtEnC;AAuEI,WAAO,CAAC,GAAC,UAAKF,WAAL,mBAAaE;EACxB;EAEAC,MAAgC;AAC9B,SAAKC,YAAY,KAAA;AACjB,WAAO,KAAKC,KAAK,YAAA;AACf,YAAM,KAAKC,QAAQ,OAAA;AACnB,aAAO,MAAM,KAAKC,WAAU;IAC9B,CAAA;EACF;EAEAC,QAA0B;AACxB,SAAKJ,YAAY,OAAA;AACjB,WAAO,KAAKC,KAAK,YAAA;AACf,YAAM,KAAKC,QAAQ,OAAA;AACnB,aAAO,MAAM,KAAKG,aAAY;IAChC,CAAA;EACF;EAEAC,SAAqC;AACnC,SAAKN,YAAY,QAAA;AACjB,WAAO,KAAKC,KAAK,YAAA;AACf,YAAM,KAAKC,QAAQ,OAAA;AACnB,aAAO,MAAM,KAAKK,cAAa;IACjC,CAAA;EACF;EAEA,MAAMC,OAAOC,QAAqC;AAChD,SAAKT,YAAY,QAAA;AACjB,WAAO,MAAM,KAAKC,KAAK,YAAA;AACrB,YAAM,KAAKC,QAAQ,OAAA;AACnB,aAAO,MAAM,KAAKQ,iBAAiBD,MAAAA;IACrC,CAAA;EACF;EAEA,MAAME,IAAIF,QAAsC;AAC9C,SAAKT,YAAY,KAAA;AACjB,WAAO,MAAM,KAAKC,KAAK,YAAA;AACrB,YAAM,KAAKC,QAAQ,OAAA;AACnB,aAAO,MAAM,KAAKU,cAAcH,MAAAA;IAClC,CAAA;EACF;EAEA,MAAMI,OAAOC,UAAyC;AACpD,SAAKd,YAAY,QAAA;AACjB,WAAO,MAAM,KAAKC,KAAK,YAAA;AACrB,YAAM,KAAKC,QAAQ,OAAA;AACnB,aAAO,MAAM,KAAKa,iBAAiBD,QAAAA;IACrC,CAAA;EACF;EAEUX,aAAuC;AAC/C,UAAM,IAAIa,MAAM,iBAAA;EAClB;EAEUX,eAAiC;AACzC,UAAM,IAAIW,MAAM,iBAAA;EAClB;EAEUT,gBAA4C;AACpD,UAAM,IAAIS,MAAM,iBAAA;EAClB;EAEUC,cAAcC,SAA4C;AAClE,UAAM,IAAIF,MAAM,iBAAA;EAClB;EAEA,MAAgBN,iBAAiBD,QAAkBb,QAA0C;AAC3F,UAAMuB,cAAavB,iCAAQuB,eAAc;AAEzC,UAAMC,gBAAgB,MAAM,KAAKH,cAAcR,MAAAA;AAE/C,QAAIU,YAAY;AACd,YAAM,KAAKE,KAAK,WAAW;QAAEZ,QAAQW;QAAeE,QAAQ;MAAK,CAAA;IACnE;AAEA,WAAOF;EACT;EAEA,MAAgBG,cAAcd,QAAkBe,WAA8D;AAC5G,UAAMC,cAAc,MAAMC,eAAeC,cAAc,MAAMH,UAAUb,IAAIF,MAAAA,CAAAA,GAAUmB,OAAO,CAAC,CAAA,EAAGC,IAAAA,MAAK;AACnG,YAAMC,WAAWrB,OAAOsB,SAASF,IAAAA;AACjC,UAAI,CAACC,UAAU;AACbE,gBAAQC,KAAK,oDAAoDJ,IAAAA,EAAM;MAEzE;AACA,aAAOC;IACT,CAAA;AAEA,UAAMI,cAAc,IAAIC,IAAIV,WAAWW,IAAI,CAAC,CAAA,EAAGP,IAAAA,MAAUA,IAAAA,CAAAA;AACzD,UAAMQ,gBAAgBZ,WAAWW,IAAI,CAAC,CAACE,OAAAA,MAAaA,OAAAA;AAEpD,UAAMC,WAAW9B,OAAOmB,OAAO,CAACC,SAAS,CAACK,YAAYM,IAAIX,IAAAA,CAAAA;AAC1D,WAAO;MAACQ;MAAeE;;EACzB;EAEA,MAAgBE,eAAehC,QAAkD;AAvKnF;AAwKI,UAAMiC,UAAUC,OAAOC,SAAQ,WAAM,KAAKF,QAAO,MAAlB,mBAAuBG,SAAQ,CAAC,CAAA;AAC/D,QAAIC,kBAAkB;SAAIrC;;AAC1B,QAAIsC,cAAc;AAClB,QAAIC,SAAoB,CAAA;AAGxB,WAAOD,cAAcL,QAAQO,UAAUH,gBAAgBG,SAAS,GAAG;AACjE,YAAM,CAACC,OAAOX,QAAAA,IAAY,MAAM,KAAKhB,cAAcuB,iBAAiBJ,QAAQK,WAAAA,CAAY;AACxFC,eAAS;WAAIA;WAAWE;;AACxBJ,wBAAkBP;AAClBQ;IACF;AACA,WAAO;MAACC;MAAQF;;EAClB;EAEUK,WAAWjC,SAA0C;AAC7D,UAAM,IAAIF,MAAM,iBAAA;EAClB;EAEA,MAAgBJ,cAAcH,QAAkBb,QAA2C;AAEzF,UAAMuB,cAAavB,iCAAQuB,eAAc;AACzC,UAAMiC,SAAS,MAAM,KAAKD,WAAW1C,MAAAA;AACrC,UAAM2B,MAAM,MAAMV,eAAe2B,UAAUD,MAAAA;AAC3C,UAAME,UAAU,MAAM5B,eAAe6B,cAAcH,MAAAA;AAEnD,UAAMf,gBAAqC,CAAA;AAC3C,UAAMmB,iBAA2B,CAAA;AACjC,eAAW3B,QAAQpB,QAAQ;AACzB,YAAMyC,QAAQd,IAAIP,IAAAA,KAASyB,QAAQzB,IAAAA;AACnC,UAAIqB,OAAO;AACTb,sBAAcoB,KAAKC,cAAcC,YAAYT,KAAAA,CAAAA;MAC/C,OAAO;AACLM,uBAAeC,KAAK5B,IAAAA;MACtB;IACF;AAEA,UAAM,CAAC+B,mBAAAA,IAAuB,MAAM,KAAKnB,eAAee,cAAAA;AAExD,QAAI,KAAK1D,kBAAkB;AACzB,YAAM,KAAKiB,iBAAiB6C,mBAAAA;IAC9B;AACA,WAAO;SAAIvB;SAAkBuB;;EAC/B;EAEUC,OAAwC;AAChD,WAAO,KAAKtE;EACd;EAEUuE,cAAcC,WAA0C;AAChE,UAAM,IAAI/C,MAAM,iBAAA;EAClB;EAEA,MAAgBD,iBAAiBD,UAAqBlB,QAA2C;AAC/F,UAAMuB,cAAavB,iCAAQuB,eAAc;AACzC,UAAM6C,kBAAiBpE,iCAAQoE,mBAAkB;AAEjD,UAAMC,mBAAmB,MAAM,KAAKH,cAAchD,QAAAA;AAElD,QAAIkD,gBAAgB;AAClB,YAAM,KAAKA,eAAeC,gBAAAA;IAC5B;AACA,QAAI9C,YAAY;AACd,YAAM,KAAKE,KAAK,YAAY;QAAEC,QAAQ;QAAMR,UAAUmD;MAAiB,CAAA;IACzE;AAEA,WAAOA;EACT;EAEA,MAAgBvB,UAAU;AA7O5B;AA8OI,SAAKlD,WAAW,KAAKA,YAAY;MAC/Bc,QAAQ,MAAM,KAAK4D,mBAAkB,gBAAKtE,WAAL,mBAAa8C,YAAb,mBAAsBpC,MAAAA;MAC3DuC,MAAM,MAAM,KAAKqB,mBAAkB,gBAAKtE,WAAL,mBAAa8C,YAAb,mBAAsBG,IAAAA;MACzDsB,OAAO,MAAM,KAAKD,mBAAkB,gBAAKtE,WAAL,mBAAa8C,YAAb,mBAAsByB,KAAAA;IAC5D;AACA,WAAOC,SAAS,KAAK5E,QAAQ;EAC/B;EAEA,MAAyB6E,aACvBC,OACAxD,UACAyD,aACmC;AA1PvC;AA2PI,UAAMC,eAAe,MAAMC,yBAAyBC,WAA2BJ,OAAOxD,QAAAA;AACtF,UAAM6D,eAAe,MAAMH,aAAaI,SAAQ;AAChDR,aAAS,MAAM,KAAKS,UAAUP,OAAOxD,UAAUyD,WAAAA,CAAAA;AAC/C,UAAMO,iBAA4B,CAAA;AAClC,QAAI,KAAKlF,OAAOmF,cAAc;AAC5B,YAAM,KAAKjB,cAAc;QAACQ;OAAM;IAClC;AAEA,YAAQK,aAAaK,QAAM;MACzB,KAAKC,yBAAyB;AAC5BH,uBAAerB,KAAI,GAAK,MAAM,KAAKtD,WAAU,CAAA;AAC7C;MACF;MACA,KAAK+E,2BAA2B;AAC9B,cAAM,KAAK7E,aAAY;AACvB;MACF;MACA,KAAK8E,4BAA4B;AAC/BL,uBAAerB,KAAI,GAAK,MAAM,KAAKlD,cAAa,CAAA;AAChD;MACF;MACA,KAAK6E,4BAA4B;AAC/B,cAAMC,gBAAsC;UAC1C5E,QAAQ;eAAK,MAAM,KAAKC,iBAAiBiE,aAAalE,MAAM;;UAC5DuE,QAAQI;QACV;AACAN,uBAAerB,KAAK4B,aAAAA;AACpB;MACF;MACA,KAAK3F,yBAAyB;AAC5B,aAAIiF,kBAAalE,WAAbkE,mBAAqB1B,QAAQ;AAC/B6B,yBAAerB,KAAI,GAAK,MAAM,KAAK7C,cAAc+D,aAAalE,MAAM,CAAA;QACtE,OAAO;AACL,gBAAMoD,OAAO,MAAM,KAAKA,KAAI;AAC5B,cAAIA;AAAMiB,2BAAerB,KAAKI,IAAAA;QAChC;AACA;MACF;MACA,KAAKyB,4BAA4B;AAC/BlB,iBAAStD,UAAU,MAAM,qBAAqByE,KAAKC,UAAUhB,aAAab,YAAW,GAAI,MAAM,CAAA,CAAA,EAAI;AACnG,cAAM8B,mBAAmB,MAAM/D,eAAegE,wBAAwB5E,UAAUwD,MAAMqB,cAAc;AACpGvB,iBACEqB,iBAAiBxC,WAAWqB,MAAMqB,eAAe1C,QACjD,MAAM,sCAAsCwC,iBAAiBxC,MAAM,OAAOqB,MAAMqB,eAAe1C,MAAM,GAAG;AAE1G,cAAM2C,uBAAuB,MAAMlE,eAAemE,cAAcJ,kBAAkB,MAAM/D,eAAeoE,SAASnB,YAAAA,CAAAA;AAChHG,uBAAerB,KAAI,GAAK,MAAM,KAAK1C,iBAAiB6E,oBAAAA,CAAAA;AAIpD,aAAKrG,uBAAuBkG,iBAAiBM,GAAG,EAAC;AACjD;MACF;MACA,SAAS;AACP,eAAO,MAAM,MAAM1B,aAAaC,OAAOxD,QAAAA;MACzC;IACF;AACA,WAAOgE;EACT;EAEA,MAAgBkB,cAAcC,QAA2BnF,UAAqB;AAC5E,WAAO,MAAMmF,OAAOpF,OAAOC,QAAAA;EAC7B;EAEA,MAAgBkD,eAAelD,UAAyC;AACtE,UAAM4B,UAAU,MAAM,KAAKA,QAAO;AAClC,WAAOwD,QACL,MAAMC,QAAQpG,IACZ4C,OAAOC,OAAOF,QAAQyB,SAAS,CAAC,CAAA,EAAG/B,IAAI,OAAO6D,WAAAA;AAC5C,aAAOA,SAAS,MAAM,KAAKD,cAAcC,QAAQnF,QAAAA,IAAYsF;IAC/D,CAAA,CAAA,CAAA,EAEFC,KAAI;EACR;EAEA,MAAcnC,kBAAkBoC,aAAuB,CAAA,GAAI;AACzD,UAAMC,mBAAmB;SAAK,MAAM,KAAKC,QAAQ;QAAEC,SAASH;MAAW,CAAA;SAAS,MAAM,KAAKE,QAAQ;QAAEE,MAAMJ;MAAW,CAAA;MAAK1E,OACzH+E,gBAAAA;AAGFvC,aACE,CAAC,KAAKzE,qBAAqB4G,iBAAiBtD,WAAWqD,WAAWrD,QAClE,MACE,qEAAqEqD,WAAW1E,OAAO,CAACJ,cACtF+E,iBAAiBnE,IAAI,CAACd,WAAW,EAAEA,OAAOmF,YAAYjF,aAAaF,OAAO1B,OAAO8G,SAASlF,UAAQ,CAAA,CAAA,GAChG;AAIR,WAAO+E,iBAAiBK,OAA0C,CAACC,MAAMvF,WAAAA;AACvEuF,WAAKvF,OAAOmF,OAAO,IAAIK,oBAAoBxF,QAAQ,MAAA;AACjDyF,4BAAoBzF,QAAQ;UAAE0F,KAAKhF;QAAQ,CAAA;AAC3C,eAAO,oDAAoDV,OAAOmF,OAAO,KAAKnF,OAAO1B,OAAO8G,IAAI,KAAKpF,OAAO1B,OAAOoF,MAAM;MAC3H,CAAA;AAEA,aAAO6B;IACT,GAAG,CAAC,CAAA;EACN;AACF;AAhTUvH;AAJH,IAAeD,oBAAf;","names":["assertEx","compact","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistCommitQuerySchema","ArchivistDeleteQuerySchema","ArchivistGetQuerySchema","ArchivistInsertQuerySchema","asArchivistInstance","isArchivistInstance","QueryBoundWitnessWrapper","PayloadHasher","AbstractModuleInstance","duplicateModules","PayloadBuilder","AbstractArchivist","AbstractModuleInstance","_lastInsertedPayload","_parents","queries","ArchivistGetQuerySchema","requireAllParents","config","_queryAccountPaths","storeParentReads","all","_noOverride","busy","started","allHandler","clear","clearHandler","commit","commitHandler","delete","hashes","deleteWithConfig","get","getWithConfig","insert","payloads","insertWithConfig","Error","deleteHandler","_hashes","emitEvents","deletedHashes","emit","module","getFromParent","archivist","foundPairs","PayloadBuilder","dataHashPairs","filter","hash","askedFor","includes","console","warn","foundHashes","Set","map","foundPayloads","payload","notfound","has","getFromParents","parents","Object","values","read","remainingHashes","parentIndex","result","length","found","getHandler","gotten","toHashMap","dataMap","toDataHashMap","notfoundHashes","push","PayloadHasher","jsonPayload","parentFoundPayloads","head","insertHandler","_payloads","writeToParents","insertedPayloads","resolveArchivists","write","assertEx","queryHandler","query","queryConfig","wrappedQuery","QueryBoundWitnessWrapper","parseQuery","queryPayload","getQuery","queryable","resultPayloads","storeQueries","schema","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistCommitQuerySchema","ArchivistDeleteQuerySchema","resultPayload","ArchivistInsertQuerySchema","JSON","stringify","resolvedPayloads","filterIncludeByDataHash","payload_hashes","payloadsWithoutQuery","filterExclude","dataHash","at","writeToParent","parent","compact","Promise","undefined","flat","archivists","archivistModules","resolve","address","name","duplicateModules","reduce","prev","asArchivistInstance","isArchivistInstance","log"]}
+{"version":3,"sources":["../../src/AbstractArchivist.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { compact } from '@xylabs/lodash'\nimport { Promisable, PromisableArray } from '@xylabs/promise'\nimport {\n  ArchivistAllQuerySchema,\n  ArchivistClearQuerySchema,\n  ArchivistCommitQuerySchema,\n  ArchivistDeleteQuery,\n  ArchivistDeleteQuerySchema,\n  ArchivistGetQuerySchema,\n  ArchivistInsertQuerySchema,\n  ArchivistInstance,\n  ArchivistModuleEventData,\n  ArchivistParams,\n  ArchivistQuery,\n  ArchivistQueryBase,\n  asArchivistInstance,\n  isArchivistInstance,\n} from '@xyo-network/archivist-model'\nimport { BoundWitness, QueryBoundWitness } from '@xyo-network/boundwitness-model'\nimport { QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-wrapper'\nimport { AbstractModuleInstance } from '@xyo-network/module-abstract'\nimport { duplicateModules, ModuleConfig, ModuleQueryHandlerResult } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, PayloadWithMeta, WithMeta } from '@xyo-network/payload-model'\n\nexport interface ActionConfig {\n  emitEvents?: boolean\n}\n\nexport interface InsertConfig extends ActionConfig {\n  writeToParents?: boolean\n}\n\nexport interface ArchivistParentInstances {\n  commit?: Record<string, ArchivistInstance>\n  read?: Record<string, ArchivistInstance>\n  write?: Record<string, ArchivistInstance>\n}\n\nexport abstract class AbstractArchivist<\n    TParams extends ArchivistParams = ArchivistParams,\n    TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n  >\n  extends AbstractModuleInstance<TParams, TEventData>\n  implements ArchivistInstance<TParams>\n{\n  private _lastInsertedPayload: Payload | undefined\n  private _parents?: ArchivistParentInstances\n\n  override get queries(): string[] {\n    return [ArchivistGetQuerySchema, ...super.queries]\n  }\n\n  get requireAllParents() {\n    return this.config.requireAllParents ?? false\n  }\n\n  protected override get _queryAccountPaths(): Record<ArchivistQueryBase['schema'], string> {\n    return {\n      'network.xyo.query.archivist.all': '1/1',\n      'network.xyo.query.archivist.clear': '1/2',\n      'network.xyo.query.archivist.commit': '1/3',\n      'network.xyo.query.archivist.delete': '1/4',\n      'network.xyo.query.archivist.get': '1/5',\n      'network.xyo.query.archivist.insert': '1/6',\n    }\n  }\n\n  protected get storeParentReads() {\n    return !!this.config?.storeParentReads\n  }\n\n  all(): PromisableArray<WithMeta<Payload>> {\n    this._noOverride('all')\n    return this.busy(async () => {\n      await this.started('throw')\n      return await this.allHandler()\n    })\n  }\n\n  clear(): Promisable<void> {\n    this._noOverride('clear')\n    return this.busy(async () => {\n      await this.started('throw')\n      return await this.clearHandler()\n    })\n  }\n\n  commit(): Promisable<WithMeta<BoundWitness>[]> {\n    this._noOverride('commit')\n    return this.busy(async () => {\n      await this.started('throw')\n      return await this.commitHandler()\n    })\n  }\n\n  async delete(hashes: string[]): Promise<string[]> {\n    this._noOverride('delete')\n    return await this.busy(async () => {\n      await this.started('throw')\n      return await this.deleteWithConfig(hashes)\n    })\n  }\n\n  async get(hashes: string[]): Promise<WithMeta<Payload>[]> {\n    this._noOverride('get')\n    return await this.busy(async () => {\n      await this.started('throw')\n      return await this.getWithConfig(hashes)\n    })\n  }\n\n  async insert(payloads: Payload[]): Promise<WithMeta<Payload>[]> {\n    this._noOverride('insert')\n    return await this.busy(async () => {\n      await this.started('throw')\n      //make sure all incoming payloads have proper $hash and $meta\n      return await this.insertWithConfig(await Promise.all(payloads.map((payload) => PayloadBuilder.build(payload, true))))\n    })\n  }\n\n  protected allHandler(): PromisableArray<WithMeta<Payload>> {\n    throw new Error('Not implemented')\n  }\n\n  protected clearHandler(): Promisable<void> {\n    throw new Error('Not implemented')\n  }\n\n  protected commitHandler(): Promisable<WithMeta<BoundWitness>[]> {\n    throw new Error('Not implemented')\n  }\n\n  protected deleteHandler(_hashes: string[]): PromisableArray<string> {\n    throw new Error('Not implemented')\n  }\n\n  protected async deleteWithConfig(hashes: string[], config?: ActionConfig): Promise<string[]> {\n    const emitEvents = config?.emitEvents ?? true\n\n    const deletedHashes = await this.deleteHandler(hashes)\n\n    if (emitEvents) {\n      await this.emit('deleted', { hashes: deletedHashes, module: this })\n    }\n\n    return deletedHashes\n  }\n\n  protected async getFromParent(hashes: string[], archivist: ArchivistInstance): Promise<[WithMeta<Payload>[], string[]]> {\n    const foundPairs = (await PayloadBuilder.dataHashPairs(await archivist.get(hashes))).filter(([, hash]) => {\n      const askedFor = hashes.includes(hash)\n      if (!askedFor) {\n        console.warn(`Parent returned payload with hash not asked for: ${hash}`)\n        //throw Error(`Parent returned payload with hash not asked for: ${hash}`)\n      }\n      return askedFor\n    })\n\n    const foundHashes = new Set(foundPairs.map(([, hash]) => hash))\n    const foundPayloads = foundPairs.map(([payload]) => payload)\n\n    const notfound = hashes.filter((hash) => !foundHashes.has(hash))\n    return [foundPayloads, notfound]\n  }\n\n  protected async getFromParents(hashes: string[]): Promise<[WithMeta<Payload>[], string[]]> {\n    const parents = Object.values((await this.parents())?.read ?? {})\n    let remainingHashes = [...hashes]\n    let parentIndex = 0\n    let result: WithMeta<Payload>[] = []\n\n    //intentionally doing this serially\n    while (parentIndex < parents.length && remainingHashes.length > 0) {\n      const [found, notfound] = await this.getFromParent(remainingHashes, parents[parentIndex])\n      result = [...result, ...found]\n      remainingHashes = notfound\n      parentIndex++\n    }\n    return [result, remainingHashes]\n  }\n\n  protected getHandler(_hashes: string[]): Promisable<PayloadWithMeta[]> {\n    throw new Error('Not implemented')\n  }\n\n  protected async getWithConfig(hashes: string[], config?: InsertConfig): Promise<PayloadWithMeta[]> {\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const emitEvents = config?.emitEvents ?? true\n    const gotten = await this.getHandler(hashes)\n    const map = await PayloadBuilder.toHashMap(gotten)\n    const dataMap = await PayloadBuilder.toDataHashMap(gotten)\n\n    const foundPayloads: PayloadWithMeta[] = []\n    const notfoundHashes: string[] = []\n    for (const hash of hashes) {\n      const found = map[hash] ?? dataMap[hash]\n      if (found) {\n        foundPayloads.push((await PayloadBuilder.build(found)) as PayloadWithMeta)\n      } else {\n        notfoundHashes.push(hash)\n      }\n    }\n\n    const [parentFoundPayloads] = await this.getFromParents(notfoundHashes)\n\n    if (this.storeParentReads) {\n      await this.insertWithConfig(parentFoundPayloads)\n    }\n    return [...foundPayloads, ...parentFoundPayloads]\n  }\n\n  protected head(): Promisable<Payload | undefined> {\n    return this._lastInsertedPayload\n  }\n\n  protected insertHandler(_payloads: Payload[]): Promise<WithMeta<Payload>[]> {\n    throw new Error('Not implemented')\n  }\n\n  protected async insertQueryHandler<T extends QueryBoundWitnessWrapper = QueryBoundWitnessWrapper>(query: T, payloads?: Payload[]) {\n    assertEx(payloads, () => `Missing payloads: ${JSON.stringify(query.payload, null, 2)}`)\n    const resolvedPayloads = await PayloadBuilder.filterIncludeByDataHash(payloads, query.payloadHashes)\n    assertEx(\n      resolvedPayloads.length === query.payloadHashes.length,\n      () => `Could not find some passed hashes [${resolvedPayloads.length} != ${query.payloadHashes.length}]`,\n    )\n    const queryPayload = await query.getQuery()\n    const payloadsWithoutQuery = await PayloadBuilder.filterExclude(resolvedPayloads, await PayloadBuilder.dataHash(queryPayload))\n    const result = await this.insertWithConfig(payloadsWithoutQuery)\n    // NOTE: There isn't an exact equivalence between what we get and what we store. Once\n    // we move to returning only inserted Payloads(/hash) instead of a BoundWitness, we\n    // can grab the actual last one\n    this._lastInsertedPayload = resolvedPayloads.at(-1)\n    return result\n  }\n\n  protected async insertWithConfig(payloads: Payload[], config?: InsertConfig): Promise<PayloadWithMeta[]> {\n    const emitEvents = config?.emitEvents ?? true\n    const writeToParents = config?.writeToParents ?? true\n\n    const insertedPayloads = await this.insertHandler(payloads)\n\n    if (writeToParents) {\n      await this.writeToParents(insertedPayloads)\n    }\n    if (emitEvents) {\n      await this.emit('inserted', { module: this, payloads: insertedPayloads })\n    }\n\n    return insertedPayloads\n  }\n\n  protected async parents() {\n    this._parents = this._parents ?? {\n      commit: await this.resolveArchivists(this.config?.parents?.commit),\n      read: await this.resolveArchivists(this.config?.parents?.read),\n      write: await this.resolveArchivists(this.config?.parents?.write),\n    }\n    return assertEx(this._parents)\n  }\n\n  protected override async queryHandler<T extends QueryBoundWitness = QueryBoundWitness, TConfig extends ModuleConfig = ModuleConfig>(\n    query: T,\n    payloads: Payload[],\n    queryConfig?: TConfig,\n  ): Promise<ModuleQueryHandlerResult> {\n    const wrappedQuery = await QueryBoundWitnessWrapper.parseQuery<ArchivistQuery>(query, payloads)\n    const queryPayload = await wrappedQuery.getQuery()\n    assertEx(await this.queryable(query, payloads, queryConfig))\n    const resultPayloads: Payload[] = []\n\n    switch (queryPayload.schema) {\n      case ArchivistAllQuerySchema: {\n        resultPayloads.push(...(await this.allHandler()))\n        break\n      }\n      case ArchivistClearQuerySchema: {\n        await this.clearHandler()\n        break\n      }\n      case ArchivistCommitQuerySchema: {\n        resultPayloads.push(...(await this.commitHandler()))\n        break\n      }\n      case ArchivistDeleteQuerySchema: {\n        const resultPayload: ArchivistDeleteQuery = {\n          hashes: [...(await this.deleteWithConfig(queryPayload.hashes))],\n          schema: ArchivistDeleteQuerySchema,\n        }\n        resultPayloads.push(resultPayload)\n        break\n      }\n      case ArchivistGetQuerySchema: {\n        if (queryPayload.hashes?.length) {\n          resultPayloads.push(...(await this.getWithConfig(queryPayload.hashes)))\n        } else {\n          const head = await this.head()\n          if (head) resultPayloads.push(head)\n        }\n        break\n      }\n      case ArchivistInsertQuerySchema: {\n        resultPayloads.push(...(await this.insertQueryHandler(wrappedQuery, payloads)))\n        break\n      }\n      default: {\n        const result = await super.queryHandler(query, payloads)\n        if (this.config.storeQueries) {\n          await this.insertHandler([query])\n        }\n        return result\n      }\n    }\n    if (this.config.storeQueries) {\n      await this.insertHandler([query])\n    }\n    return resultPayloads\n  }\n\n  protected async writeToParent(parent: ArchivistInstance, payloads: Payload[]) {\n    return await parent.insert(payloads)\n  }\n\n  protected async writeToParents(payloads: Payload[]): Promise<PayloadWithMeta[]> {\n    const parents = await this.parents()\n    return compact(\n      await Promise.all(\n        Object.values(parents.write ?? {}).map(async (parent) => {\n          return parent ? await this.writeToParent(parent, payloads) : undefined\n        }),\n      ),\n    ).flat()\n  }\n\n  private async resolveArchivists(archivists: string[] = []) {\n    const archivistModules = [...(await this.resolve({ address: archivists })), ...(await this.resolve({ name: archivists }))].filter(\n      duplicateModules,\n    )\n\n    assertEx(\n      !this.requireAllParents || archivistModules.length === archivists.length,\n      () =>\n        `Failed to find some archivists (set allRequired to false if ok): [${archivists.filter((archivist) =>\n          archivistModules.map((module) => !(module.address === archivist || module.config.name === archivist)),\n        )}]`,\n    )\n\n    // eslint-disable-next-line unicorn/no-array-reduce\n    return archivistModules.reduce<Record<string, ArchivistInstance>>((prev, module) => {\n      prev[module.address] = asArchivistInstance(module, () => {\n        isArchivistInstance(module, { log: console })\n        return `Unable to cast resolved module to an archivist: [${module.address}, ${module.config.name}, ${module.config.schema})}]`\n      })\n\n      return prev\n    }, {})\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,eAAe;AAExB,SACEC,yBACAC,2BACAC,4BAEAC,4BACAC,yBACAC,4BAMAC,qBACAC,2BACK;AAEP,SAASC,gCAAgC;AACzC,SAASC,8BAA8B;AACvC,SAASC,wBAAgE;AACzE,SAASC,sBAAsB;AAiBxB,IAAeC,qBAAf,MAAeA,2BAIZC,uBAAAA;EAGAC;EACAC;EAER,IAAaC,UAAoB;AAC/B,WAAO;MAACC;SAA4B,MAAMD;;EAC5C;EAEA,IAAIE,oBAAoB;AACtB,WAAO,KAAKC,OAAOD,qBAAqB;EAC1C;EAEA,IAAuBE,qBAAmE;AACxF,WAAO;MACL,mCAAmC;MACnC,qCAAqC;MACrC,sCAAsC;MACtC,sCAAsC;MACtC,mCAAmC;MACnC,sCAAsC;IACxC;EACF;EAEA,IAAcC,mBAAmB;AArEnC;AAsEI,WAAO,CAAC,GAAC,UAAKF,WAAL,mBAAaE;EACxB;EAEAC,MAA0C;AACxC,SAAKC,YAAY,KAAA;AACjB,WAAO,KAAKC,KAAK,YAAA;AACf,YAAM,KAAKC,QAAQ,OAAA;AACnB,aAAO,MAAM,KAAKC,WAAU;IAC9B,CAAA;EACF;EAEAC,QAA0B;AACxB,SAAKJ,YAAY,OAAA;AACjB,WAAO,KAAKC,KAAK,YAAA;AACf,YAAM,KAAKC,QAAQ,OAAA;AACnB,aAAO,MAAM,KAAKG,aAAY;IAChC,CAAA;EACF;EAEAC,SAA+C;AAC7C,SAAKN,YAAY,QAAA;AACjB,WAAO,KAAKC,KAAK,YAAA;AACf,YAAM,KAAKC,QAAQ,OAAA;AACnB,aAAO,MAAM,KAAKK,cAAa;IACjC,CAAA;EACF;EAEA,MAAMC,OAAOC,QAAqC;AAChD,SAAKT,YAAY,QAAA;AACjB,WAAO,MAAM,KAAKC,KAAK,YAAA;AACrB,YAAM,KAAKC,QAAQ,OAAA;AACnB,aAAO,MAAM,KAAKQ,iBAAiBD,MAAAA;IACrC,CAAA;EACF;EAEA,MAAME,IAAIF,QAAgD;AACxD,SAAKT,YAAY,KAAA;AACjB,WAAO,MAAM,KAAKC,KAAK,YAAA;AACrB,YAAM,KAAKC,QAAQ,OAAA;AACnB,aAAO,MAAM,KAAKU,cAAcH,MAAAA;IAClC,CAAA;EACF;EAEA,MAAMI,OAAOC,UAAmD;AAC9D,SAAKd,YAAY,QAAA;AACjB,WAAO,MAAM,KAAKC,KAAK,YAAA;AACrB,YAAM,KAAKC,QAAQ,OAAA;AAEnB,aAAO,MAAM,KAAKa,iBAAiB,MAAMC,QAAQjB,IAAIe,SAASG,IAAI,CAACC,YAAYC,eAAeC,MAAMF,SAAS,IAAA,CAAA,CAAA,CAAA;IAC/G,CAAA;EACF;EAEUf,aAAiD;AACzD,UAAM,IAAIkB,MAAM,iBAAA;EAClB;EAEUhB,eAAiC;AACzC,UAAM,IAAIgB,MAAM,iBAAA;EAClB;EAEUd,gBAAsD;AAC9D,UAAM,IAAIc,MAAM,iBAAA;EAClB;EAEUC,cAAcC,SAA4C;AAClE,UAAM,IAAIF,MAAM,iBAAA;EAClB;EAEA,MAAgBX,iBAAiBD,QAAkBb,QAA0C;AAC3F,UAAM4B,cAAa5B,iCAAQ4B,eAAc;AAEzC,UAAMC,gBAAgB,MAAM,KAAKH,cAAcb,MAAAA;AAE/C,QAAIe,YAAY;AACd,YAAM,KAAKE,KAAK,WAAW;QAAEjB,QAAQgB;QAAeE,QAAQ;MAAK,CAAA;IACnE;AAEA,WAAOF;EACT;EAEA,MAAgBG,cAAcnB,QAAkBoB,WAAwE;AACtH,UAAMC,cAAc,MAAMX,eAAeY,cAAc,MAAMF,UAAUlB,IAAIF,MAAAA,CAAAA,GAAUuB,OAAO,CAAC,CAAA,EAAGC,IAAAA,MAAK;AACnG,YAAMC,WAAWzB,OAAO0B,SAASF,IAAAA;AACjC,UAAI,CAACC,UAAU;AACbE,gBAAQC,KAAK,oDAAoDJ,IAAAA,EAAM;MAEzE;AACA,aAAOC;IACT,CAAA;AAEA,UAAMI,cAAc,IAAIC,IAAIT,WAAWb,IAAI,CAAC,CAAA,EAAGgB,IAAAA,MAAUA,IAAAA,CAAAA;AACzD,UAAMO,gBAAgBV,WAAWb,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;AAEpD,UAAMuB,WAAWhC,OAAOuB,OAAO,CAACC,SAAS,CAACK,YAAYI,IAAIT,IAAAA,CAAAA;AAC1D,WAAO;MAACO;MAAeC;;EACzB;EAEA,MAAgBE,eAAelC,QAA4D;AAvK7F;AAwKI,UAAMmC,UAAUC,OAAOC,SAAQ,WAAM,KAAKF,QAAO,MAAlB,mBAAuBG,SAAQ,CAAC,CAAA;AAC/D,QAAIC,kBAAkB;SAAIvC;;AAC1B,QAAIwC,cAAc;AAClB,QAAIC,SAA8B,CAAA;AAGlC,WAAOD,cAAcL,QAAQO,UAAUH,gBAAgBG,SAAS,GAAG;AACjE,YAAM,CAACC,OAAOX,QAAAA,IAAY,MAAM,KAAKb,cAAcoB,iBAAiBJ,QAAQK,WAAAA,CAAY;AACxFC,eAAS;WAAIA;WAAWE;;AACxBJ,wBAAkBP;AAClBQ;IACF;AACA,WAAO;MAACC;MAAQF;;EAClB;EAEUK,WAAW9B,SAAkD;AACrE,UAAM,IAAIF,MAAM,iBAAA;EAClB;EAEA,MAAgBT,cAAcH,QAAkBb,QAAmD;AAEjG,UAAM4B,cAAa5B,iCAAQ4B,eAAc;AACzC,UAAM8B,SAAS,MAAM,KAAKD,WAAW5C,MAAAA;AACrC,UAAMQ,MAAM,MAAME,eAAeoC,UAAUD,MAAAA;AAC3C,UAAME,UAAU,MAAMrC,eAAesC,cAAcH,MAAAA;AAEnD,UAAMd,gBAAmC,CAAA;AACzC,UAAMkB,iBAA2B,CAAA;AACjC,eAAWzB,QAAQxB,QAAQ;AACzB,YAAM2C,QAAQnC,IAAIgB,IAAAA,KAASuB,QAAQvB,IAAAA;AACnC,UAAImB,OAAO;AACTZ,sBAAcmB,KAAM,MAAMxC,eAAeC,MAAMgC,KAAAA,CAAAA;MACjD,OAAO;AACLM,uBAAeC,KAAK1B,IAAAA;MACtB;IACF;AAEA,UAAM,CAAC2B,mBAAAA,IAAuB,MAAM,KAAKjB,eAAee,cAAAA;AAExD,QAAI,KAAK5D,kBAAkB;AACzB,YAAM,KAAKiB,iBAAiB6C,mBAAAA;IAC9B;AACA,WAAO;SAAIpB;SAAkBoB;;EAC/B;EAEUC,OAAwC;AAChD,WAAO,KAAKtE;EACd;EAEUuE,cAAcC,WAAoD;AAC1E,UAAM,IAAI1C,MAAM,iBAAA;EAClB;EAEA,MAAgB2C,mBAAkFC,OAAUnD,UAAsB;AAChIoD,aAASpD,UAAU,MAAM,qBAAqBqD,KAAKC,UAAUH,MAAM/C,SAAS,MAAM,CAAA,CAAA,EAAI;AACtF,UAAMmD,mBAAmB,MAAMlD,eAAemD,wBAAwBxD,UAAUmD,MAAMM,aAAa;AACnGL,aACEG,iBAAiBlB,WAAWc,MAAMM,cAAcpB,QAChD,MAAM,sCAAsCkB,iBAAiBlB,MAAM,OAAOc,MAAMM,cAAcpB,MAAM,GAAG;AAEzG,UAAMqB,eAAe,MAAMP,MAAMQ,SAAQ;AACzC,UAAMC,uBAAuB,MAAMvD,eAAewD,cAAcN,kBAAkB,MAAMlD,eAAeyD,SAASJ,YAAAA,CAAAA;AAChH,UAAMtB,SAAS,MAAM,KAAKnC,iBAAiB2D,oBAAAA;AAI3C,SAAKnF,uBAAuB8E,iBAAiBQ,GAAG,EAAC;AACjD,WAAO3B;EACT;EAEA,MAAgBnC,iBAAiBD,UAAqBlB,QAAmD;AACvG,UAAM4B,cAAa5B,iCAAQ4B,eAAc;AACzC,UAAMsD,kBAAiBlF,iCAAQkF,mBAAkB;AAEjD,UAAMC,mBAAmB,MAAM,KAAKjB,cAAchD,QAAAA;AAElD,QAAIgE,gBAAgB;AAClB,YAAM,KAAKA,eAAeC,gBAAAA;IAC5B;AACA,QAAIvD,YAAY;AACd,YAAM,KAAKE,KAAK,YAAY;QAAEC,QAAQ;QAAMb,UAAUiE;MAAiB,CAAA;IACzE;AAEA,WAAOA;EACT;EAEA,MAAgBnC,UAAU;AA9P5B;AA+PI,SAAKpD,WAAW,KAAKA,YAAY;MAC/Bc,QAAQ,MAAM,KAAK0E,mBAAkB,gBAAKpF,WAAL,mBAAagD,YAAb,mBAAsBtC,MAAAA;MAC3DyC,MAAM,MAAM,KAAKiC,mBAAkB,gBAAKpF,WAAL,mBAAagD,YAAb,mBAAsBG,IAAAA;MACzDkC,OAAO,MAAM,KAAKD,mBAAkB,gBAAKpF,WAAL,mBAAagD,YAAb,mBAAsBqC,KAAAA;IAC5D;AACA,WAAOf,SAAS,KAAK1E,QAAQ;EAC/B;EAEA,MAAyB0F,aACvBjB,OACAnD,UACAqE,aACmC;AA3QvC;AA4QI,UAAMC,eAAe,MAAMC,yBAAyBC,WAA2BrB,OAAOnD,QAAAA;AACtF,UAAM0D,eAAe,MAAMY,aAAaX,SAAQ;AAChDP,aAAS,MAAM,KAAKqB,UAAUtB,OAAOnD,UAAUqE,WAAAA,CAAAA;AAC/C,UAAMK,iBAA4B,CAAA;AAElC,YAAQhB,aAAaiB,QAAM;MACzB,KAAKC,yBAAyB;AAC5BF,uBAAe7B,KAAI,GAAK,MAAM,KAAKxD,WAAU,CAAA;AAC7C;MACF;MACA,KAAKwF,2BAA2B;AAC9B,cAAM,KAAKtF,aAAY;AACvB;MACF;MACA,KAAKuF,4BAA4B;AAC/BJ,uBAAe7B,KAAI,GAAK,MAAM,KAAKpD,cAAa,CAAA;AAChD;MACF;MACA,KAAKsF,4BAA4B;AAC/B,cAAMC,gBAAsC;UAC1CrF,QAAQ;eAAK,MAAM,KAAKC,iBAAiB8D,aAAa/D,MAAM;;UAC5DgF,QAAQI;QACV;AACAL,uBAAe7B,KAAKmC,aAAAA;AACpB;MACF;MACA,KAAKpG,yBAAyB;AAC5B,aAAI8E,kBAAa/D,WAAb+D,mBAAqBrB,QAAQ;AAC/BqC,yBAAe7B,KAAI,GAAK,MAAM,KAAK/C,cAAc4D,aAAa/D,MAAM,CAAA;QACtE,OAAO;AACL,gBAAMoD,OAAO,MAAM,KAAKA,KAAI;AAC5B,cAAIA;AAAM2B,2BAAe7B,KAAKE,IAAAA;QAChC;AACA;MACF;MACA,KAAKkC,4BAA4B;AAC/BP,uBAAe7B,KAAI,GAAK,MAAM,KAAKK,mBAAmBoB,cAActE,QAAAA,CAAAA;AACpE;MACF;MACA,SAAS;AACP,cAAMoC,SAAS,MAAM,MAAMgC,aAAajB,OAAOnD,QAAAA;AAC/C,YAAI,KAAKlB,OAAOoG,cAAc;AAC5B,gBAAM,KAAKlC,cAAc;YAACG;WAAM;QAClC;AACA,eAAOf;MACT;IACF;AACA,QAAI,KAAKtD,OAAOoG,cAAc;AAC5B,YAAM,KAAKlC,cAAc;QAACG;OAAM;IAClC;AACA,WAAOuB;EACT;EAEA,MAAgBS,cAAcC,QAA2BpF,UAAqB;AAC5E,WAAO,MAAMoF,OAAOrF,OAAOC,QAAAA;EAC7B;EAEA,MAAgBgE,eAAehE,UAAiD;AAC9E,UAAM8B,UAAU,MAAM,KAAKA,QAAO;AAClC,WAAOuD,QACL,MAAMnF,QAAQjB,IACZ8C,OAAOC,OAAOF,QAAQqC,SAAS,CAAC,CAAA,EAAGhE,IAAI,OAAOiF,WAAAA;AAC5C,aAAOA,SAAS,MAAM,KAAKD,cAAcC,QAAQpF,QAAAA,IAAYsF;IAC/D,CAAA,CAAA,CAAA,EAEFC,KAAI;EACR;EAEA,MAAcrB,kBAAkBsB,aAAuB,CAAA,GAAI;AACzD,UAAMC,mBAAmB;SAAK,MAAM,KAAKC,QAAQ;QAAEC,SAASH;MAAW,CAAA;SAAS,MAAM,KAAKE,QAAQ;QAAEE,MAAMJ;MAAW,CAAA;MAAKtE,OACzH2E,gBAAAA;AAGFzC,aACE,CAAC,KAAKvE,qBAAqB4G,iBAAiBpD,WAAWmD,WAAWnD,QAClE,MACE,qEAAqEmD,WAAWtE,OAAO,CAACH,cACtF0E,iBAAiBtF,IAAI,CAACU,WAAW,EAAEA,OAAO8E,YAAY5E,aAAaF,OAAO/B,OAAO8G,SAAS7E,UAAQ,CAAA,CAAA,GAChG;AAIR,WAAO0E,iBAAiBK,OAA0C,CAACC,MAAMlF,WAAAA;AACvEkF,WAAKlF,OAAO8E,OAAO,IAAIK,oBAAoBnF,QAAQ,MAAA;AACjDoF,4BAAoBpF,QAAQ;UAAEqF,KAAK5E;QAAQ,CAAA;AAC3C,eAAO,oDAAoDT,OAAO8E,OAAO,KAAK9E,OAAO/B,OAAO8G,IAAI,KAAK/E,OAAO/B,OAAO6F,MAAM;MAC3H,CAAA;AAEA,aAAOoB;IACT,GAAG,CAAC,CAAA;EACN;AACF;AA3TUvH;AAJH,IAAeD,oBAAf;","names":["assertEx","compact","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistCommitQuerySchema","ArchivistDeleteQuerySchema","ArchivistGetQuerySchema","ArchivistInsertQuerySchema","asArchivistInstance","isArchivistInstance","QueryBoundWitnessWrapper","AbstractModuleInstance","duplicateModules","PayloadBuilder","AbstractArchivist","AbstractModuleInstance","_lastInsertedPayload","_parents","queries","ArchivistGetQuerySchema","requireAllParents","config","_queryAccountPaths","storeParentReads","all","_noOverride","busy","started","allHandler","clear","clearHandler","commit","commitHandler","delete","hashes","deleteWithConfig","get","getWithConfig","insert","payloads","insertWithConfig","Promise","map","payload","PayloadBuilder","build","Error","deleteHandler","_hashes","emitEvents","deletedHashes","emit","module","getFromParent","archivist","foundPairs","dataHashPairs","filter","hash","askedFor","includes","console","warn","foundHashes","Set","foundPayloads","notfound","has","getFromParents","parents","Object","values","read","remainingHashes","parentIndex","result","length","found","getHandler","gotten","toHashMap","dataMap","toDataHashMap","notfoundHashes","push","parentFoundPayloads","head","insertHandler","_payloads","insertQueryHandler","query","assertEx","JSON","stringify","resolvedPayloads","filterIncludeByDataHash","payloadHashes","queryPayload","getQuery","payloadsWithoutQuery","filterExclude","dataHash","at","writeToParents","insertedPayloads","resolveArchivists","write","queryHandler","queryConfig","wrappedQuery","QueryBoundWitnessWrapper","parseQuery","queryable","resultPayloads","schema","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistCommitQuerySchema","ArchivistDeleteQuerySchema","resultPayload","ArchivistInsertQuerySchema","storeQueries","writeToParent","parent","compact","undefined","flat","archivists","archivistModules","resolve","address","name","duplicateModules","reduce","prev","asArchivistInstance","isArchivistInstance","log"]}

package/package.json

@@ -10,17 +10,16 @@
     "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js/issues"
   },
   "dependencies": {
-    "@xylabs/assert": "^2.14.0",
-    "@xylabs/lodash": "^2.14.0",
-    "@xylabs/promise": "^2.14.0",
-    "@xyo-network/archivist-model": "~2.90.1",
-    "@xyo-network/boundwitness-builder": "~2.90.1",
-    "@xyo-network/boundwitness-model": "~2.90.1",
-    "@xyo-network/hash": "~2.90.1",
-    "@xyo-network/module-abstract": "~2.90.1",
-    "@xyo-network/module-model": "~2.90.1",
-    "@xyo-network/payload-builder": "~2.90.1",
-    "@xyo-network/payload-model": "~2.90.1"
+    "@xylabs/assert": "^2.14.1",
+    "@xylabs/lodash": "^2.14.1",
+    "@xylabs/promise": "^2.14.1",
+    "@xyo-network/archivist-model": "~2.90.3",
+    "@xyo-network/boundwitness-model": "~2.90.3",
+    "@xyo-network/boundwitness-wrapper": "~2.90.3",
+    "@xyo-network/module-abstract": "~2.90.3",
+    "@xyo-network/module-model": "~2.90.3",
+    "@xyo-network/payload-builder": "~2.90.3",
+    "@xyo-network/payload-model": "~2.90.3"
   },
   "devDependencies": {
     "@xylabs/ts-scripts-yarn3": "^3.2.42",
@@ -66,6 +65,6 @@
   },
   "sideEffects": false,
   "types": "dist/node/index.d.ts",
-  "version": "2.90.1",
+  "version": "2.90.3",
   "type": "module"
 }

package/src/AbstractArchivist.ts

@@ -17,13 +17,12 @@ import {
   asArchivistInstance,
   isArchivistInstance,
 } from '@xyo-network/archivist-model'
-import { QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-builder'
 import { BoundWitness, QueryBoundWitness } from '@xyo-network/boundwitness-model'
-import { PayloadHasher } from '@xyo-network/hash'
+import { QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-wrapper'
 import { AbstractModuleInstance } from '@xyo-network/module-abstract'
 import { duplicateModules, ModuleConfig, ModuleQueryHandlerResult } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
-import { Payload, WithMeta } from '@xyo-network/payload-model'
+import { Payload, PayloadWithMeta, WithMeta } from '@xyo-network/payload-model'
 
 export interface ActionConfig {
   emitEvents?: boolean
@@ -72,7 +71,7 @@ export abstract class AbstractArchivist<
     return !!this.config?.storeParentReads
   }
 
-  all(): PromisableArray<Payload> {
+  all(): PromisableArray<WithMeta<Payload>> {
     this._noOverride('all')
     return this.busy(async () => {
       await this.started('throw')
@@ -88,7 +87,7 @@ export abstract class AbstractArchivist<
     })
   }
 
-  commit(): Promisable<BoundWitness[]> {
+  commit(): Promisable<WithMeta<BoundWitness>[]> {
     this._noOverride('commit')
     return this.busy(async () => {
       await this.started('throw')
@@ -104,7 +103,7 @@ export abstract class AbstractArchivist<
     })
   }
 
-  async get(hashes: string[]): Promise<Payload[]> {
+  async get(hashes: string[]): Promise<WithMeta<Payload>[]> {
     this._noOverride('get')
     return await this.busy(async () => {
       await this.started('throw')
@@ -112,15 +111,16 @@ export abstract class AbstractArchivist<
     })
   }
 
-  async insert(payloads: Payload[]): Promise<Payload[]> {
+  async insert(payloads: Payload[]): Promise<WithMeta<Payload>[]> {
     this._noOverride('insert')
     return await this.busy(async () => {
       await this.started('throw')
-      return await this.insertWithConfig(payloads)
+      //make sure all incoming payloads have proper $hash and $meta
+      return await this.insertWithConfig(await Promise.all(payloads.map((payload) => PayloadBuilder.build(payload, true))))
     })
   }
 
-  protected allHandler(): PromisableArray<Payload> {
+  protected allHandler(): PromisableArray<WithMeta<Payload>> {
     throw new Error('Not implemented')
   }
 
@@ -128,7 +128,7 @@ export abstract class AbstractArchivist<
     throw new Error('Not implemented')
   }
 
-  protected commitHandler(): Promisable<BoundWitness[]> {
+  protected commitHandler(): Promisable<WithMeta<BoundWitness>[]> {
     throw new Error('Not implemented')
   }
 
@@ -148,7 +148,7 @@ export abstract class AbstractArchivist<
     return deletedHashes
   }
 
-  protected async getFromParent(hashes: string[], archivist: ArchivistInstance): Promise<[Payload[], string[]]> {
+  protected async getFromParent(hashes: string[], archivist: ArchivistInstance): Promise<[WithMeta<Payload>[], string[]]> {
     const foundPairs = (await PayloadBuilder.dataHashPairs(await archivist.get(hashes))).filter(([, hash]) => {
       const askedFor = hashes.includes(hash)
       if (!askedFor) {
@@ -165,11 +165,11 @@ export abstract class AbstractArchivist<
     return [foundPayloads, notfound]
   }
 
-  protected async getFromParents(hashes: string[]): Promise<[Payload[], string[]]> {
+  protected async getFromParents(hashes: string[]): Promise<[WithMeta<Payload>[], string[]]> {
     const parents = Object.values((await this.parents())?.read ?? {})
     let remainingHashes = [...hashes]
     let parentIndex = 0
-    let result: Payload[] = []
+    let result: WithMeta<Payload>[] = []
 
     //intentionally doing this serially
     while (parentIndex < parents.length && remainingHashes.length > 0) {
@@ -181,23 +181,23 @@ export abstract class AbstractArchivist<
     return [result, remainingHashes]
   }
 
-  protected getHandler(_hashes: string[]): Promisable<Payload[]> {
+  protected getHandler(_hashes: string[]): Promisable<PayloadWithMeta[]> {
     throw new Error('Not implemented')
   }
 
-  protected async getWithConfig(hashes: string[], config?: InsertConfig): Promise<Payload[]> {
+  protected async getWithConfig(hashes: string[], config?: InsertConfig): Promise<PayloadWithMeta[]> {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     const emitEvents = config?.emitEvents ?? true
     const gotten = await this.getHandler(hashes)
     const map = await PayloadBuilder.toHashMap(gotten)
     const dataMap = await PayloadBuilder.toDataHashMap(gotten)
 
-    const foundPayloads: WithMeta<Payload>[] = []
+    const foundPayloads: PayloadWithMeta[] = []
     const notfoundHashes: string[] = []
     for (const hash of hashes) {
       const found = map[hash] ?? dataMap[hash]
       if (found) {
-        foundPayloads.push(PayloadHasher.jsonPayload(found) as WithMeta<Payload>)
+        foundPayloads.push((await PayloadBuilder.build(found)) as PayloadWithMeta)
       } else {
         notfoundHashes.push(hash)
       }
@@ -215,11 +215,28 @@ export abstract class AbstractArchivist<
     return this._lastInsertedPayload
   }
 
-  protected insertHandler(_payloads: Payload[]): Promise<Payload[]> {
+  protected insertHandler(_payloads: Payload[]): Promise<WithMeta<Payload>[]> {
     throw new Error('Not implemented')
   }
 
-  protected async insertWithConfig(payloads: Payload[], config?: InsertConfig): Promise<Payload[]> {
+  protected async insertQueryHandler<T extends QueryBoundWitnessWrapper = QueryBoundWitnessWrapper>(query: T, payloads?: Payload[]) {
+    assertEx(payloads, () => `Missing payloads: ${JSON.stringify(query.payload, null, 2)}`)
+    const resolvedPayloads = await PayloadBuilder.filterIncludeByDataHash(payloads, query.payloadHashes)
+    assertEx(
+      resolvedPayloads.length === query.payloadHashes.length,
+      () => `Could not find some passed hashes [${resolvedPayloads.length} != ${query.payloadHashes.length}]`,
+    )
+    const queryPayload = await query.getQuery()
+    const payloadsWithoutQuery = await PayloadBuilder.filterExclude(resolvedPayloads, await PayloadBuilder.dataHash(queryPayload))
+    const result = await this.insertWithConfig(payloadsWithoutQuery)
+    // NOTE: There isn't an exact equivalence between what we get and what we store. Once
+    // we move to returning only inserted Payloads(/hash) instead of a BoundWitness, we
+    // can grab the actual last one
+    this._lastInsertedPayload = resolvedPayloads.at(-1)
+    return result
+  }
+
+  protected async insertWithConfig(payloads: Payload[], config?: InsertConfig): Promise<PayloadWithMeta[]> {
     const emitEvents = config?.emitEvents ?? true
     const writeToParents = config?.writeToParents ?? true
 
@@ -253,9 +270,6 @@ export abstract class AbstractArchivist<
     const queryPayload = await wrappedQuery.getQuery()
     assertEx(await this.queryable(query, payloads, queryConfig))
     const resultPayloads: Payload[] = []
-    if (this.config.storeQueries) {
-      await this.insertHandler([query])
-    }
 
     switch (queryPayload.schema) {
       case ArchivistAllQuerySchema: {
@@ -288,24 +302,20 @@ export abstract class AbstractArchivist<
         break
       }
       case ArchivistInsertQuerySchema: {
-        assertEx(payloads, () => `Missing payloads: ${JSON.stringify(wrappedQuery.jsonPayload(), null, 2)}`)
-        const resolvedPayloads = await PayloadBuilder.filterIncludeByDataHash(payloads, query.payload_hashes)
-        assertEx(
-          resolvedPayloads.length === query.payload_hashes.length,
-          () => `Could not find some passed hashes [${resolvedPayloads.length} != ${query.payload_hashes.length}]`,
-        )
-        const payloadsWithoutQuery = await PayloadBuilder.filterExclude(resolvedPayloads, await PayloadBuilder.dataHash(queryPayload))
-        resultPayloads.push(...(await this.insertWithConfig(payloadsWithoutQuery)))
-        // NOTE: There isn't an exact equivalence between what we get and what we store. Once
-        // we move to returning only inserted Payloads(/hash) instead of a BoundWitness, we
-        // can grab the actual last one
-        this._lastInsertedPayload = resolvedPayloads.at(-1)
+        resultPayloads.push(...(await this.insertQueryHandler(wrappedQuery, payloads)))
         break
       }
       default: {
-        return await super.queryHandler(query, payloads)
+        const result = await super.queryHandler(query, payloads)
+        if (this.config.storeQueries) {
+          await this.insertHandler([query])
+        }
+        return result
       }
     }
+    if (this.config.storeQueries) {
+      await this.insertHandler([query])
+    }
     return resultPayloads
   }
 
@@ -313,7 +323,7 @@ export abstract class AbstractArchivist<
     return await parent.insert(payloads)
   }
 
-  protected async writeToParents(payloads: Payload[]): Promise<Payload[]> {
+  protected async writeToParents(payloads: Payload[]): Promise<PayloadWithMeta[]> {
     const parents = await this.parents()
     return compact(
       await Promise.all(