@xyo-network/sdk 7.3.1 → 7.4.1

package/dist/neutral/archivist-abstract.mjs

@@ -1,7 +1,6 @@
 // src/modules/archivist-abstract/AbstractArchivist.ts
 import {
   assertEx,
-  difference,
   exists,
   isNull,
   isUndefined
@@ -40,6 +39,8 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
   _parentArchivists;
   _payloadCountGauge;
   _payloadCountMeter;
+  activeCacheMutations = 0;
+  cacheGeneration = 0;
   // do not override this!  It is meant to get the this.defaultNextLimitSetting and work if it is overridden
   static get defaultNextLimit() {
     return this.defaultNextLimitSetting;
@@ -102,9 +103,7 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
         await this.globalReentrancyMutex?.acquire();
         return await this.busy(async () => {
           await this.startedAsync("throw");
-          await this.clearHandler();
-          this.reportPayloadCount();
-          await this.emit("cleared", { mod: this });
+          await this.clearWithConfig();
         });
       } finally {
         this.globalReentrancyMutex?.release();
@@ -188,6 +187,7 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
   async insert(payloads) {
     this._noOverride("insert");
     this.isSupportedQuery(ArchivistInsertQuerySchema, "insert");
+    const snapshot = this.snapshotPayloads(payloads);
     return await this.spanAsync(`${this.id}|insert`, async () => {
       if (this.reentrancy?.scope === "global" && this.reentrancy.action === "skip" && this.globalReentrancyMutex?.isLocked()) {
         return [];
@@ -196,7 +196,7 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
         await this.globalReentrancyMutex?.acquire();
         return await this.busy(async () => {
           await this.startedAsync("throw");
-          return await this.insertWithConfig(PayloadBuilder.omitStorageMeta(payloads));
+          return await this.insertWithConfig(snapshot);
         });
       } finally {
         this.globalReentrancyMutex?.release();
@@ -206,7 +206,7 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
   async insertQuery(payloads, account) {
     this._noOverride("insertQuery");
     const queryPayload = { schema: ArchivistInsertQuerySchema };
-    return await this.sendQueryRaw(queryPayload, payloads, account);
+    return await this.sendQueryRaw(queryPayload, this.snapshotPayloads(payloads), account);
   }
   async next(options) {
     this._noOverride("next");
@@ -261,6 +261,13 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
   clearHandler() {
     throw new Error(NOT_IMPLEMENTED);
   }
+  async clearWithConfig(config) {
+    await this.withCacheMutation(async () => {
+      await this.withStorageMutation(async () => await this.clearHandler());
+      this.reportPayloadCount();
+      if (config?.emitEvents ?? true) await this.emit("cleared", { mod: this });
+    });
+  }
   commitHandler() {
     throw new Error(NOT_IMPLEMENTED);
   }
@@ -268,18 +275,20 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
     throw new Error(NOT_IMPLEMENTED);
   }
   async deleteWithConfig(hashes, config) {
-    const emitEvents = config?.emitEvents ?? true;
-    const payloads = await this.deleteHandler(hashes);
-    const hashesDeleted = payloads.map((p) => p._hash);
-    if (emitEvents) {
-      await this.emit("deleted", {
-        hashes: hashesDeleted,
-        payloads,
-        mod: this
-      });
-    }
-    this.reportPayloadCount();
-    return payloads;
+    return await this.withCacheMutation(async () => {
+      const emitEvents = config?.emitEvents ?? true;
+      const payloads = await this.withStorageMutation(async () => await this.deleteHandler(hashes));
+      const hashesDeleted = payloads.map((p) => p._hash);
+      if (emitEvents) {
+        await this.emit("deleted", {
+          hashes: hashesDeleted,
+          payloads,
+          mod: this
+        });
+      }
+      this.reportPayloadCount();
+      return payloads;
+    });
   }
   generateStats() {
     return {
@@ -287,18 +296,18 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
       schema: ArchivistStatsPayloadSchema
     };
   }
+  /**
+   * Inspect local stored roots without consulting public projections, caches or
+   * parents. Backends whose getHandler synthesizes views must override this hook
+   * to read their primary record keys directly.
+   */
+  async getExistingLocalPayloads(hashes) {
+    return await this.exactRepresentations(await this.getHandler(hashes), hashes);
+  }
   async getFromParent(hashes, archivist) {
-    const foundPairs = (await PayloadBuilder.dataHashPairs(await archivist.get(hashes))).filter(([, hash]) => {
-      const askedFor = hashes.includes(hash);
-      if (!askedFor) {
-        console.warn(`Parent returned payload with hash not asked for: ${hash}`);
-      }
-      return askedFor;
-    });
-    const foundHashes = new Set(foundPairs.map(([, hash]) => hash));
-    const foundPayloads = foundPairs.map(([payload]) => payload);
-    const notfound = hashes.filter((hash) => !foundHashes.has(hash));
-    return [foundPayloads, notfound];
+    const foundPayloads = await this.requestedRepresentations(await archivist.get(hashes), hashes);
+    const foundHashes = new Set(foundPayloads.map((payload) => payload._hash));
+    return [foundPayloads, hashes.filter((hash) => !foundHashes.has(hash))];
   }
   async getFromParents(hashes) {
     const parents = Object.values((await this.parentArchivists())?.read ?? {});
@@ -319,58 +328,25 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
   getHandler(_hashes) {
     throw new Error(NOT_IMPLEMENTED);
   }
-  // eslint-disable-next-line max-statements
   async getWithConfig(hashes, _config) {
-    const requestedHashes = new Set(hashes);
-    const cache = this._getCache;
-    let fromCache = [];
-    let remainingHashes = [...requestedHashes];
-    if (cache !== void 0) {
-      fromCache = hashes.map((hash) => cache.get(hash)).filter(exists);
-      remainingHashes = hashes.filter((hash) => !fromCache.some((payload) => payload?._hash === hash || payload?._dataHash === hash));
-    }
-    const fromGet = await this.getHandler([...remainingHashes]);
-    const gotten = [...fromCache, ...fromGet].toSorted(PayloadBuilder.compareStorageMeta);
-    const foundPayloads = [];
-    const foundHashes = /* @__PURE__ */ new Set();
-    for (const payload of gotten) {
-      const map = {
-        [payload._hash]: payload,
-        [payload._dataHash]: payload
-      };
-      for (const [key, payload2] of Object.entries(map)) {
-        let requestedPayloadFound = false;
-        const hash = key;
-        if (requestedHashes.has(hash) && !foundHashes.has(hash)) {
-          requestedPayloadFound = true;
-          foundHashes.add(hash);
-        }
-        if (requestedPayloadFound) foundPayloads.push(payload2);
-      }
-    }
-    const notFoundHashes = [...difference(requestedHashes, foundHashes)];
-    const [parentFoundPayloads] = await this.getFromParents(notFoundHashes);
-    if (this.storeParentReads) {
+    const requestedHashes = [...new Set(hashes)];
+    const generation = this.cacheGeneration;
+    const cache = this.activeCacheMutations === 0 ? this._getCache : void 0;
+    const fromCache = await this.requestedRepresentations(requestedHashes.map((hash) => cache?.get(hash)).filter(exists), requestedHashes);
+    const cachedHashes = new Set(fromCache.map((payload) => payload._hash));
+    const remainingHashes = requestedHashes.filter((hash) => !cachedHashes.has(hash));
+    const fromGet = await this.requestedRepresentations(await this.getHandler(remainingHashes), remainingHashes);
+    const foundHashes = new Set([...fromCache, ...fromGet].map((payload) => payload._hash));
+    const [parentFoundPayloads] = await this.getFromParents(requestedHashes.filter((hash) => !foundHashes.has(hash)));
+    if (this.storeParentReads && parentFoundPayloads.length > 0) {
       await this.insertWithConfig(parentFoundPayloads);
     }
-    const result = this.omitClientMetaForDataHashes(
-      hashes,
-      PayloadBuilder.omitPrivateStorageMeta([
-        ...foundPayloads,
-        ...parentFoundPayloads
-      ]).toSorted(PayloadBuilder.compareStorageMeta)
-    );
-    if (cache !== void 0) {
-      for (const payload of fromGet) {
-        cache.set(payload._hash, payload);
-        cache.set(payload._dataHash, payload);
-      }
-      for (const payload of parentFoundPayloads) {
-        cache.set(payload._hash, payload);
-        cache.set(payload._dataHash, payload);
-      }
+    if (this.activeCacheMutations === 0 && generation === this.cacheGeneration) {
+      for (const payload of [...fromGet, ...parentFoundPayloads]) cache?.set(payload._hash, payload);
     }
-    return result;
+    return PayloadBuilder.omitPrivateStorageMeta(
+      [...fromCache, ...fromGet, ...parentFoundPayloads].toSorted(PayloadBuilder.compareStorageMeta)
+    );
   }
   insertHandler(_payloads) {
     throw new Error(NOT_IMPLEMENTED);
@@ -388,27 +364,38 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
     return result;
   }
   async insertWithConfig(payloads, config) {
-    const emitEvents = config?.emitEvents ?? true;
-    const writeToParents = config?.writeToParents ?? true;
-    const withStorageMeta = await PayloadBuilder.addStorageMeta(payloads);
-    const hashes = withStorageMeta.map((p) => p._hash);
-    const existingPayloads = await this.getWithConfig(hashes);
-    const existingHashes = new Set(existingPayloads.map((p) => p._hash));
-    const payloadsToInsert = withStorageMeta.filter((p) => !existingHashes.has(p._hash));
-    const insertedPayloads = await this.insertHandler(payloadsToInsert);
-    if (writeToParents) {
-      await this.writeToParents(insertedPayloads);
-    }
-    if (emitEvents) {
-      await this.emit("inserted", {
-        mod: this,
-        payloads: insertedPayloads,
-        outPayloads: insertedPayloads,
-        inPayloads: payloads
+    const sanitizedPayloads = this.snapshotPayloads(payloads);
+    return await this.withCacheMutation(async () => {
+      const emitEvents = config?.emitEvents ?? true;
+      const writeToParents = config?.writeToParents ?? true;
+      await this.validateInsertPayloads(sanitizedPayloads);
+      const withStorageMeta = await PayloadBuilder.addStorageMeta(sanitizedPayloads);
+      const hashes = [...new Set(withStorageMeta.map((payload) => payload._hash))];
+      const { existingPayloads, insertedPayloads } = await this.withStorageMutation(async () => {
+        const existingPayloads2 = await this.getExistingLocalPayloads(hashes);
+        await this.repairExistingPayloads(existingPayloads2);
+        const existingHashes = new Set(existingPayloads2.map((payload) => payload._hash));
+        const payloadsToInsert = withStorageMeta.filter((payload) => {
+          if (existingHashes.has(payload._hash)) return false;
+          existingHashes.add(payload._hash);
+          return true;
+        });
+        return { existingPayloads: existingPayloads2, insertedPayloads: await this.insertHandler(payloadsToInsert) };
       });
-    }
-    this.reportPayloadCount();
-    return PayloadBuilder.omitPrivateStorageMeta(insertedPayloads);
+      if (writeToParents) {
+        await this.writeToParents([...existingPayloads, ...insertedPayloads]);
+      }
+      if (emitEvents) {
+        await this.emit("inserted", {
+          mod: this,
+          payloads: insertedPayloads,
+          outPayloads: insertedPayloads,
+          inPayloads: payloads
+        });
+      }
+      this.reportPayloadCount();
+      return PayloadBuilder.omitPrivateStorageMeta(insertedPayloads);
+    });
   }
   nextHandler(_options) {
     throw new Error(NOT_IMPLEMENTED);
@@ -433,7 +420,7 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
   // eslint-disable-next-line max-statements
   async queryHandler(query, payloads, queryConfig) {
     const sanitizedQuery = PayloadBuilder.omitStorageMeta(query);
-    const sanitizedPayloads = PayloadBuilder.omitStorageMeta(payloads);
+    const sanitizedPayloads = this.snapshotPayloads(payloads);
     const wrappedQuery = QueryBoundWitnessWrapper.parseQuery(sanitizedQuery, sanitizedPayloads);
     const queryPayload = await wrappedQuery.getQuery();
     assertEx(await this.queryable(sanitizedQuery, sanitizedPayloads, queryConfig));
@@ -444,7 +431,7 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
         break;
       }
       case ArchivistClearQuerySchema: {
-        await this.clearHandler();
+        await this.clearWithConfig();
         break;
       }
       case ArchivistCommitQuerySchema: {
@@ -487,6 +474,10 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
     }
     return PayloadBuilder.omitPrivateStorageMeta(resultPayloads);
   }
+  /** Restore derived indexes for a stored root without rewriting its body. */
+  repairExistingPayloads(_payloads) {
+    return void 0;
+  }
   reportPayloadCount() {
     this._noOverride("reportPayloadCount");
     const gauge = this.payloadCountGauge;
@@ -513,8 +504,27 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
   async stateHandler() {
     return [...await super.stateHandler(), await this.generateStats()];
   }
+  /** Enforce backend admission before lookup, repair, insertion or parent writes. */
+  validateInsertPayloads(_payloads) {
+    return void 0;
+  }
+  /** Serialize local storage changes without holding a lock across callbacks or parents. */
+  async withStorageMutation(operation) {
+    return await operation();
+  }
   async writeToParent(parent, payloads) {
-    return await parent.insert(PayloadBuilder.omitStorageMeta(payloads));
+    if (payloads.length === 0) return [];
+    const sanitized = PayloadBuilder.omitStorageMeta(payloads);
+    const hashes = [...new Set(await Promise.all(sanitized.map((payload) => PayloadBuilder.hash(payload))))];
+    const inserted = await this.exactRepresentations(await parent.insert(sanitized), hashes);
+    const insertedHashes = new Set(inserted.map((payload) => payload._hash));
+    const missing = hashes.filter((hash) => !insertedHashes.has(hash));
+    const existing = missing.length > 0 ? await this.exactRepresentations(await parent.get(missing), missing) : [];
+    const acknowledged = new Set([...inserted, ...existing].map((payload) => payload._hash));
+    if (hashes.some((hash) => !acknowledged.has(hash))) {
+      throw new Error("Write parent did not acknowledge all requested payloads");
+    }
+    return [...inserted, ...existing];
   }
   async writeToParents(payloads) {
     const parents = await this.parentArchivists();
@@ -524,16 +534,30 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
       })
     )).filter(exists).flat();
   }
-  omitClientMetaForDataHashes(hashes, payloads) {
-    return payloads.map((payload) => {
-      if (hashes.includes(payload._dataHash) && !hashes.includes(payload._hash)) {
-        const result = PayloadBuilder.omitClientMeta(payload);
-        result._hash = result._dataHash;
-        return result;
-      } else {
-        return payload;
+  async exactRepresentations(payloads, hashes) {
+    const requested = new Set(hashes);
+    const found = /* @__PURE__ */ new Map();
+    for (const payload of payloads) {
+      await this.verifyPayloadIdentity(payload);
+      if (requested.has(payload._hash)) found.set(payload._hash, payload);
+    }
+    return [...found.values()];
+  }
+  async requestedRepresentations(payloads, hashes) {
+    const requested = new Set(hashes);
+    const result = /* @__PURE__ */ new Map();
+    const sorted = payloads.toSorted(PayloadBuilder.compareStorageMeta);
+    for (const payload of sorted) {
+      await this.verifyPayloadIdentity(payload);
+      if (requested.has(payload._hash) && !result.has(payload._hash)) result.set(payload._hash, payload);
+    }
+    for (const payload of sorted) {
+      if (requested.has(payload._dataHash) && !result.has(payload._dataHash)) {
+        const dataOnly = PayloadBuilder.omitClientMeta(payload);
+        result.set(payload._dataHash, { ...dataOnly, _hash: payload._dataHash });
       }
-    });
+    }
+    return [...result.values()];
   }
   async resolveArchivists(archivists = [], archivistInstances) {
     const archivistModules = (await Promise.all(archivists.map((archivist) => this.resolve(archivist)))).filter(exists).filter(duplicateModules);
@@ -553,9 +577,70 @@ var AbstractArchivist = class _AbstractArchivist extends AbstractModuleInstance
       return prev;
     }, archivistInstancesMap);
   }
+  snapshotPayloads(payloads) {
+    return JSON.parse(JSON.stringify(PayloadBuilder.omitStorageMeta(payloads)));
+  }
+  async verifyPayloadIdentity(payload) {
+    const canonicalPayload = PayloadBuilder.omitStorageMeta(payload);
+    const [hash, dataHash] = await Promise.all([
+      PayloadBuilder.hash(canonicalPayload),
+      PayloadBuilder.dataHash(canonicalPayload)
+    ]);
+    if (hash !== payload._hash || dataHash !== payload._dataHash) {
+      throw new Error("Archivist returned payload with invalid hash metadata");
+    }
+  }
+  async withCacheMutation(operation) {
+    this.activeCacheMutations++;
+    this.cacheGeneration++;
+    this._getCache?.clear();
+    try {
+      return await operation();
+    } finally {
+      this.activeCacheMutations--;
+      this.cacheGeneration++;
+      this._getCache?.clear();
+    }
+  }
 };
+
+// src/modules/archivist-abstract/payloadAdmission.ts
+import { PayloadBuilder as PayloadBuilder2 } from "@xyo-network/sdk-protocol";
+function validByteLimit(value) {
+  return Number.isSafeInteger(value) && value > 0;
+}
+function validatePayloadAdmissionPolicy(policy) {
+  if (!policy || !Array.isArray(policy.allowedSchemas) || policy.allowedSchemas.some((schema) => typeof schema !== "string" || schema.length === 0 || schema.trim() !== schema) || !validByteLimit(policy.maxPayloadBytes)) {
+    throw new TypeError("Invalid payload admission policy");
+  }
+  if (policy.schemaMaxPayloadBytes !== void 0) {
+    if (policy.schemaMaxPayloadBytes === null || typeof policy.schemaMaxPayloadBytes !== "object" || Array.isArray(policy.schemaMaxPayloadBytes)) {
+      throw new TypeError("Invalid schema payload byte limits");
+    }
+    for (const [schema, limit] of Object.entries(policy.schemaMaxPayloadBytes)) {
+      if (!policy.allowedSchemas.includes(schema) || !validByteLimit(limit)) {
+        throw new TypeError("Invalid schema payload byte limits");
+      }
+    }
+  }
+}
+function validatePayloadAdmission(payloads, policy) {
+  validatePayloadAdmissionPolicy(policy);
+  const allowed = new Set(policy.allowedSchemas);
+  for (const payload of payloads) {
+    if (!allowed.has(payload.schema)) throw new Error("Payload schema is not admitted by this archivist");
+    const schemaLimit = policy.schemaMaxPayloadBytes && Object.hasOwn(policy.schemaMaxPayloadBytes, payload.schema) ? policy.schemaMaxPayloadBytes[payload.schema] : policy.maxPayloadBytes;
+    const limit = Math.min(policy.maxPayloadBytes, schemaLimit);
+    const json = JSON.stringify(PayloadBuilder2.omitStorageMeta(payload));
+    if (new TextEncoder().encode(json).byteLength > limit) {
+      throw new Error("Payload exceeds this archivist byte limit");
+    }
+  }
+}
 export {
   AbstractArchivist,
-  StorageClassLabel
+  StorageClassLabel,
+  validatePayloadAdmission,
+  validatePayloadAdmissionPolicy
 };
 //# sourceMappingURL=archivist-abstract.mjs.map