@vrtmrz/livesync-commonlib 0.1.32 → 0.1.34

package/README.md

@@ -26,6 +26,7 @@ The package is ESM-only and declares Node 20 or later. Browser entry points are
 | -------------------------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- |
 | `@vrtmrz/livesync-commonlib`                       | `DirectFileManipulator` for integrations which access CouchDB directly            | Deliberately small, but its high-level lifecycle and file-operation semantics are not final |
 | `@vrtmrz/livesync-commonlib/context`               | Instance-owned events, translation injection, and host-neutral standard-I/O types | Focused, package-tested pre-1.0 contract                                                    |
+| `@vrtmrz/livesync-commonlib/hashing`               | Chunk ID calculation with the database's hash manager                            | Focused, package-tested pre-1.0 contract                                                    |
 | `@vrtmrz/livesync-commonlib/browser`               | Rooted File System Access API storage                                             | Focused, package-tested pre-1.0 contract                                                    |
 | `@vrtmrz/livesync-commonlib/node`                  | Rooted Node storage, Node standard I/O, and selected Node capabilities            | Supported platform façade; keeps Node-only dependencies behind an explicit boundary         |
 | `@vrtmrz/livesync-commonlib/p2p`                   | Host-composed P2P service views and composition helpers                           | Focused, package-tested pre-1.0 contract with a temporary compatibility façade              |
@@ -118,6 +119,12 @@ See [the settings lifecycle guide](docs/settings-lifecycle.md) before initialisi
 
 The focused Setup URI entry offers an Ephemeral mode bound to the current fixed seven-day window and a Persistent mode compatible with existing readers. Both use the same URI structure. See [the Setup URI sharing guide](docs/setup-uri.md) for the availability boundary, import behaviour, and compatibility limits.
 
+## Chunk ID calculation
+
+The `/hashing` entry exports `HashManager` and its `HashManagerCoreOptions` type. It uses the same implementation as database writes, including legacy algorithms and independent ID keys. Hosts supply their settings service and await `initialise()` before calling `computeHash(piece)`.
+
+`computeHash` returns the Chunk hash, including the `+` prefix when encryption is enabled; the database adds its `h:` namespace separately. Reuse a manager for repeated calculations with the same configuration, create a new manager when changing the legacy algorithm or passphrase, and call `clearCaches()` when retiring it. This releases retained key preparation; it does not erase existing settings or in-flight operations. The entry performs ID calculation and does not write Chunks to a database.
+
 ## Remote connection profiles
 
 New profile-management code can create or update a CouchDB, Object Storage, or P2P connection through the focused remote-configurations entry. Profile IDs provide identity, display names remain editable presentation, and the active selection is explicit. Older flat connection fields remain a runtime projection and an import-compatibility boundary rather than the preferred persistence model.

package/dist/common/idDerivation.d.ts

@@ -9,5 +9,7 @@ export declare function formatIdRecoveryCode(key: string): string;
 export declare function deriveOrImportIdKey(input: string): Promise<string>;
 /** Return the independent ID key, rejecting an incomplete new-mode setting. */
 export declare function configuredIdKey(settings: Pick<EncryptionSettings, "idDerivationVersion" | "idDerivationKey">): string | false;
-/** Derive distinct identifiers for content and document paths from one saved key. */
+/** Prepare a purpose-separated Chunk key for repeated ID calculations. */
+export declare function createChunkIdGenerator(key: string): Promise<(value: string) => Promise<string>>;
+/** Derive purpose-separated IDs; repeated Chunk calculations should reuse a prepared generator. */
 export declare function computeKeyedId(key: string, purpose: "chunk" | "document" | "peer-agreement" | "remote-agreement", value: string): Promise<string>;

package/dist/common/idDerivation.js

@@ -1,10 +1,13 @@
 import { getWebCrypto } from "../mods.js";
 import { hexStringToUint8Array, uint8ArrayToHexString } from "../string_and_binary/convert.js";
+import { xxhashNew } from "../string_and_binary/hash.js";
 const ID_DERIVATION_VERSION = 1;
 const ID_RECOVERY_CODE_PREFIX = "sls-id-v1:";
 const SOURCE_SALT = new TextEncoder().encode("self-hosted-livesync:id-source:v1");
 const SOURCE_ITERATIONS = 31e4;
 const HEX_KEY = /^[0-9a-f]{64}$/u;
+const CHUNK_KEY_CONTEXT = new TextEncoder().encode("self-hosted-livesync:id-v1:chunk-key:xxhash64");
+const CHUNK_ID_PREFIX = "self-hosted-livesync:id-v1:chunk:xxhash64\0";
 async function deriveIdKey(source) {
   if (source.length === 0) {
     throw new Error("An ID source is required.");
@@ -50,7 +53,30 @@ function configuredIdKey(settings) {
   }
   return key;
 }
+async function createChunkIdGenerator(key) {
+  if (!HEX_KEY.test(key)) throw new Error("The configured ID key is invalid.");
+  const crypto = await getWebCrypto();
+  const masterKey = await crypto.subtle.importKey(
+    "raw",
+    hexStringToUint8Array(key),
+    { name: "HMAC", hash: "SHA-256" },
+    false,
+    ["sign"]
+  );
+  const derivedKey = await crypto.subtle.sign("HMAC", masterKey, CHUNK_KEY_CONTEXT);
+  const chunkKey = await crypto.subtle.importKey("raw", derivedKey, { name: "HMAC", hash: "SHA-256" }, false, [
+    "sign"
+  ]);
+  const xxhash = await xxhashNew();
+  const encoder = new TextEncoder();
+  return async (value) => {
+    const digest = xxhash.h64(value).toString(16).padStart(16, "0");
+    const id = await crypto.subtle.sign("HMAC", chunkKey, encoder.encode(CHUNK_ID_PREFIX + digest));
+    return uint8ArrayToHexString(new Uint8Array(id));
+  };
+}
 async function computeKeyedId(key, purpose, value) {
+  if (purpose === "chunk") return (await createChunkIdGenerator(key))(value);
   if (!HEX_KEY.test(key)) throw new Error("The configured ID key is invalid.");
   const crypto = await getWebCrypto();
   const hmacKey = await crypto.subtle.importKey(
@@ -69,6 +95,7 @@ export {
   ID_RECOVERY_CODE_PREFIX,
   computeKeyedId,
   configuredIdKey,
+  createChunkIdGenerator,
   deriveIdKey,
   deriveOrImportIdKey,
   formatIdRecoveryCode

package/dist/hashing.d.ts

@@ -0,0 +1,3 @@
+/** Chunk ID calculation using the same hash manager as database writes. */
+export { HashManager } from "./managers/HashManager/HashManager.js";
+export type { HashManagerCoreOptions } from "./managers/HashManager/HashManagerCore.js";

package/dist/hashing.js

@@ -0,0 +1,4 @@
+import { HashManager } from "./managers/HashManager/HashManager.js";
+export {
+  HashManager
+};

package/dist/managers/ChunkFetcher.d.ts

@@ -15,6 +15,13 @@ export declare class ChunkFetcher {
     get chunkManager(): ChunkManager;
     queue: DocumentID[];
     private readonly pendingClaims;
+    private reportedPendingCount;
+    private reportedFetchCounts;
+    private requestTimer;
+    private requestTimerDueAt;
+    private finiteActivityWasActive;
+    private finiteCompletionVersion;
+    private readonly stopObservingDelivery;
     private destroyed;
     get interval(): number;
     get concurrency(): number;
@@ -23,12 +30,20 @@ export declare class ChunkFetcher {
     destroy(): void;
     onEventHandler: (ids: DocumentID[]) => void;
     onEvent(ids: DocumentID[]): void;
+    private clearRequestTimer;
+    private scheduleRequest;
     private ensureClaims;
     private removePendingDelivery;
+    private updatePendingCount;
     private onClaimStalled;
     private settleClaim;
+    private getActiveClaimIds;
     private touchClaims;
     private waitForActivityBoundary;
+    private waitBeforeRemoteRequest;
+    private resolveLocallyAvailableRetries;
+    private logFetchedChunks;
+    private applyMissingChunkResults;
     /**
      * Processing requests
      */
@@ -40,5 +55,6 @@ export declare class ChunkFetcher {
      */
     previousRequestTime: number;
     canRequestMore(): boolean;
+    private isReady;
     requestMissingChunks(): Promise<void>;
 }

package/dist/managers/ChunkFetcher.js

@@ -5,6 +5,11 @@ import { DEFAULT_SETTINGS } from "../common/types.js";
 import {} from "./ChunkManager.js";
 import { compatGlobal } from "../common/coreEnvFunctions.js";
 import { DEFAULT_CHUNK_DELIVERY_STALL_TIMEOUT_MS } from "./ChunkDeliveryCoordinator.js";
+import { chunkFetchCounts, collectingChunks } from "../mock_and_interop/stores.js";
+import {
+  classifyFetchedChunks,
+  decideMissingChunkAction
+} from "./chunkFetcher.helper.js";
 const EVENT_MISSING_CHUNKS = "missingChunks";
 const EVENT_MISSING_CHUNK_REMOTE = "missingChunkRemote";
 const EVENT_CHUNK_FETCHED = "chunkFetched";
@@ -19,6 +24,13 @@ class ChunkFetcher {
   }
   queue = [];
   pendingClaims = /* @__PURE__ */ new Map();
+  reportedPendingCount = 0;
+  reportedFetchCounts = { initial: 0, retrying: 0 };
+  requestTimer;
+  requestTimerDueAt = 0;
+  finiteActivityWasActive;
+  finiteCompletionVersion = 0;
+  stopObservingDelivery;
   destroyed = false;
   get interval() {
     const settings = this.options.settingService.currentSettings();
@@ -31,6 +43,18 @@ class ChunkFetcher {
   abort = new AbortController();
   constructor(options) {
     this.options = options;
+    this.finiteActivityWasActive = this.chunkManager.deliveryCoordinator.isFiniteReplicationActive();
+    this.stopObservingDelivery = this.chunkManager.deliveryCoordinator.onChanged(() => {
+      const active = this.chunkManager.deliveryCoordinator.isFiniteReplicationActive();
+      const completed = this.finiteActivityWasActive && !active;
+      this.finiteActivityWasActive = active;
+      if (!completed) return;
+      this.finiteCompletionVersion++;
+      for (const pending of this.pendingClaims.values()) {
+        if (pending.missingResponses > 0 && !pending.inFlight) pending.nextRequestAt = Date.now();
+      }
+      this.scheduleRequest();
+    });
     this.chunkManager.addListener(EVENT_MISSING_CHUNKS, this.onEventHandler, {
       signal: this.abort.signal
     });
@@ -39,9 +63,12 @@ class ChunkFetcher {
     if (this.destroyed) return;
     this.destroyed = true;
     this.abort.abort();
+    this.stopObservingDelivery();
+    this.clearRequestTimer();
     this.queue = [];
-    const pendingDeliveries = new Set(this.pendingClaims.values());
+    const pendingDeliveries = new Set([...this.pendingClaims.values()].map((pending) => pending.delivery));
     this.pendingClaims.clear();
+    this.updatePendingCount();
     for (const pending of pendingDeliveries) {
       pending.resolveActivityBoundary(false);
       pending.claim.release();
@@ -52,9 +79,35 @@ class ChunkFetcher {
     if (this.destroyed) return;
     const claimedIds = this.ensureClaims(ids);
     this.queue = unique([...this.queue, ...claimedIds]);
-    if (this.canRequestMore()) {
-      compatGlobal.setTimeout(() => void this.requestMissingChunks(), 1);
+    this.scheduleRequest(1);
+  }
+  clearRequestTimer() {
+    if (this.requestTimer !== void 0) compatGlobal.clearTimeout(this.requestTimer);
+    this.requestTimer = void 0;
+  }
+  scheduleRequest(minimumDelayMs = 0) {
+    if (this.destroyed || this.currentProcessing >= this.concurrency || this.queue.length === 0) {
+      this.clearRequestTimer();
+      return;
+    }
+    let nextRequestAt = Infinity;
+    for (const id of this.queue) {
+      const pending = this.pendingClaims.get(id);
+      if (!pending?.inFlight) nextRequestAt = Math.min(nextRequestAt, pending?.nextRequestAt ?? 0);
     }
+    if (nextRequestAt === Infinity) {
+      this.clearRequestTimer();
+      return;
+    }
+    const now = Date.now();
+    const dueAt = Math.max(nextRequestAt, now + minimumDelayMs);
+    if (this.requestTimer !== void 0 && this.requestTimerDueAt <= dueAt) return;
+    this.clearRequestTimer();
+    this.requestTimerDueAt = dueAt;
+    this.requestTimer = compatGlobal.setTimeout(() => {
+      this.requestTimer = void 0;
+      void this.requestMissingChunks();
+    }, dueAt - now);
   }
   ensureClaims(ids) {
     const unclaimedIds = unique(ids.filter((id) => !this.pendingClaims.has(id)));
@@ -71,8 +124,9 @@ class ChunkFetcher {
       resolveActivityBoundary: activityBoundary.resolve
     };
     for (const id of unclaimedIds) {
-      this.pendingClaims.set(id, pending);
+      this.pendingClaims.set(id, { delivery: pending, missingResponses: 0, nextRequestAt: 0, inFlight: false });
     }
+    this.updatePendingCount();
     void this.options.replicatorService.runBoundedRemoteActivity(
       () => {
         pending.claim.touch();
@@ -92,45 +146,142 @@ class ChunkFetcher {
   }
   removePendingDelivery(pending, ids) {
     for (const id of ids) {
-      if (this.pendingClaims.get(id) === pending) {
+      if (this.pendingClaims.get(id)?.delivery === pending) {
         this.pendingClaims.delete(id);
       }
     }
     const removed = new Set(ids);
     this.queue = this.queue.filter((id) => !(removed.has(id) && !this.pendingClaims.has(id)));
+    this.updatePendingCount();
+    this.scheduleRequest();
+  }
+  updatePendingCount() {
+    const nextCount = this.pendingClaims.size;
+    const difference = nextCount - this.reportedPendingCount;
+    this.reportedPendingCount = nextCount;
+    if (difference !== 0) collectingChunks.value += difference;
+    const nextFetchCounts = { initial: 0, retrying: 0 };
+    for (const pending of this.pendingClaims.values()) {
+      if (pending.missingResponses > 0) nextFetchCounts.retrying++;
+      else nextFetchCounts.initial++;
+    }
+    const initialDifference = nextFetchCounts.initial - this.reportedFetchCounts.initial;
+    const retryingDifference = nextFetchCounts.retrying - this.reportedFetchCounts.retrying;
+    this.reportedFetchCounts = nextFetchCounts;
+    if (initialDifference !== 0 || retryingDifference !== 0) {
+      chunkFetchCounts.value = {
+        initial: chunkFetchCounts.value.initial + initialDifference,
+        retrying: chunkFetchCounts.value.retrying + retryingDifference
+      };
+    }
   }
   onClaimStalled(pending, stalledIds) {
     pending.resolveActivityBoundary(false);
     this.removePendingDelivery(pending, stalledIds);
     Logger(`Chunk delivery stalled for the following IDs: ${stalledIds.join(", ")}`, LOG_LEVEL_VERBOSE);
   }
-  settleClaim(id) {
-    const pending = this.pendingClaims.get(id);
-    if (!pending) return;
+  settleClaim(id, pending) {
+    if (!pending || this.pendingClaims.get(id) !== pending) return;
     this.pendingClaims.delete(id);
-    pending.claim.settle(id);
+    pending.delivery.claim.settle(id);
+    this.updatePendingCount();
   }
-  touchClaims(ids) {
-    const pendingDeliveries = new Set(
-      ids.map((id) => this.pendingClaims.get(id)).filter((pending) => pending !== void 0)
-    );
+  getActiveClaimIds(ids, claims) {
+    return ids.filter((id) => claims.has(id) && this.pendingClaims.get(id) === claims.get(id));
+  }
+  touchClaims(ids, claims) {
+    const pendingDeliveries = new Set(this.getActiveClaimIds(ids, claims).map((id) => claims.get(id).delivery));
     for (const pending of pendingDeliveries) {
       pending.claim.touch();
     }
   }
-  async waitForActivityBoundary(ids) {
-    const pendingDeliveries = new Set(
-      ids.map((id) => this.pendingClaims.get(id)).filter((pending) => pending !== void 0)
-    );
+  async waitForActivityBoundary(claims) {
+    const pendingDeliveries = new Set([...claims.values()].map((pending) => pending.delivery));
     const entered = new Map(
       await Promise.all(
         [...pendingDeliveries].map(async (pending) => [pending, await pending.activityBoundaryEntered])
       )
     );
-    return ids.filter((id) => {
-      const pending = this.pendingClaims.get(id);
-      return pending !== void 0 && entered.get(pending) === true;
+    return [...claims].flatMap(
+      ([id, pending]) => this.pendingClaims.get(id) === pending && entered.get(pending.delivery) === true ? [id] : []
+    );
+  }
+  async waitBeforeRemoteRequest(ids, claims, localCheckCompletionVersion) {
+    this.touchClaims(ids, claims);
+    while (!this.destroyed) {
+      const activeIds = this.getActiveClaimIds(ids, claims);
+      if (activeIds.length === 0) return [];
+      if (localCheckCompletionVersion !== this.finiteCompletionVersion) {
+        localCheckCompletionVersion = this.finiteCompletionVersion;
+        await this.resolveLocallyAvailableRetries(claims);
+        continue;
+      }
+      const now = Date.now();
+      const timeToWait = this.previousRequestTime + this.interval - now;
+      if (timeToWait > 0) {
+        await delay(timeToWait);
+        continue;
+      }
+      this.previousRequestTime = now;
+      this.touchClaims(activeIds, claims);
+      return activeIds;
+    }
+    return [];
+  }
+  async resolveLocallyAvailableRetries(claims) {
+    const retryIds = this.getActiveClaimIds([...claims.keys()], claims).filter(
+      (id) => claims.get(id).missingResponses > 0
+    );
+    if (retryIds.length === 0) return;
+    const localChunks = await this.chunkManager.read(retryIds, {
+      preventRemoteRequest: true,
+      skipCache: true,
+      waitForDelivery: false
     });
+    for (const chunk of localChunks) {
+      if (!chunk) continue;
+      this.chunkManager.emitEvent(EVENT_CHUNK_FETCHED, chunk);
+      this.settleClaim(chunk._id, claims.get(chunk._id));
+    }
+  }
+  logFetchedChunks(requestedIds, { chunks, invalidChunks }) {
+    if (invalidChunks.length > 0) {
+      Logger(
+        `Some fetched chunks are invalid and will be ignored: (${invalidChunks.length} / ${chunks.length + invalidChunks.length}).`,
+        LOG_LEVEL_VERBOSE
+      );
+      for (const chunk of invalidChunks) {
+        Logger(`Invalid chunk: ${JSON.stringify(chunk)}`, LOG_LEVEL_VERBOSE);
+      }
+    }
+    if (chunks.length === 0) {
+      Logger(`No valid chunks were found for the following IDs: ${requestedIds.join(", ")}`);
+    }
+  }
+  applyMissingChunkResults(missingIds, requestClaims, requestCompletionVersion) {
+    for (const chunkID of this.getActiveClaimIds(missingIds, requestClaims)) {
+      const pending = requestClaims.get(chunkID);
+      const completedDuringRequest = requestCompletionVersion !== this.finiteCompletionVersion;
+      const action = decideMissingChunkAction({
+        missingResponses: pending.missingResponses,
+        finiteReplicationActive: this.chunkManager.deliveryCoordinator.isFiniteReplicationActive(),
+        completedDuringRequest
+      });
+      if (action.kind === "missing") {
+        this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
+        this.settleClaim(chunkID, pending);
+      } else {
+        pending.missingResponses = action.missingResponses;
+        pending.inFlight = false;
+        pending.nextRequestAt = Date.now() + action.delayMs;
+        this.queue.push(chunkID);
+        Logger(
+          `Remote chunk is not available yet; retrying ${chunkID} in ${action.delayMs} ms.`,
+          LOG_LEVEL_VERBOSE
+        );
+      }
+    }
+    this.updatePendingCount();
   }
   /**
    * Processing requests
@@ -143,30 +294,39 @@ class ChunkFetcher {
    */
   previousRequestTime = 0;
   canRequestMore() {
-    return this.currentProcessing < this.concurrency && this.queue.length > 0;
+    return this.currentProcessing < this.concurrency && this.queue.some((id) => this.isReady(id));
+  }
+  isReady(id) {
+    const pending = this.pendingClaims.get(id);
+    return !pending || !pending.inFlight && pending.nextRequestAt <= Date.now();
   }
   async requestMissingChunks() {
     if (this.destroyed || !this.canRequestMore()) {
+      this.scheduleRequest();
       return;
     }
     let requestIDs = [];
+    const requestClaims = /* @__PURE__ */ new Map();
     try {
-      let isValidChunk2 = function(chunk) {
-        return chunk && typeof chunk?._id === "string" && typeof chunk?.data === "string";
-      };
-      var isValidChunk = isValidChunk2;
       this.currentProcessing++;
-      requestIDs = this.queue.splice(0, BATCH_SIZE);
+      requestIDs = unique(this.queue.filter((id) => this.isReady(id))).slice(0, BATCH_SIZE);
+      const selectedIds = new Set(requestIDs);
+      this.queue = this.queue.filter((id) => !selectedIds.has(id));
       this.ensureClaims(requestIDs);
-      requestIDs = await this.waitForActivityBoundary(requestIDs);
+      for (const id of requestIDs) {
+        const pending = this.pendingClaims.get(id);
+        if (pending) {
+          pending.inFlight = true;
+          requestClaims.set(id, pending);
+        }
+      }
+      this.scheduleRequest();
+      requestIDs = await this.waitForActivityBoundary(requestClaims);
       if (requestIDs.length === 0) return;
-      this.touchClaims(requestIDs);
-      const now = Date.now();
-      const timeSinceLastRequest = now - this.previousRequestTime;
-      this.previousRequestTime = now;
-      const timeToWait = Math.max(this.interval - timeSinceLastRequest, 0);
-      if (timeToWait > 0) await delay(timeToWait);
-      this.touchClaims(requestIDs);
+      const localCheckCompletionVersion = this.finiteCompletionVersion;
+      await this.resolveLocallyAvailableRetries(requestClaims);
+      let pendingIDs = await this.waitBeforeRemoteRequest(requestIDs, requestClaims, localCheckCompletionVersion);
+      if (pendingIDs.length === 0) return;
       const replicator = this.options.replicatorService.getActiveReplicator();
       if (!replicator) {
         Logger("No active replicator was found to request missing chunks.");
@@ -174,58 +334,46 @@ class ChunkFetcher {
       }
       if (!canFetchRemoteChunks(replicator)) {
         Logger("The active replicator does not support fetching individual remote chunks.");
-        for (const chunkID of requestIDs) {
+        for (const chunkID of pendingIDs) {
           this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
         }
         return;
       }
-      const fetched = await replicator.fetchRemoteChunks(requestIDs, false);
-      this.touchClaims(requestIDs);
+      const requestCompletionVersion = this.finiteCompletionVersion;
+      const fetched = await replicator.fetchRemoteChunks(pendingIDs, false);
+      pendingIDs = this.getActiveClaimIds(pendingIDs, requestClaims);
+      this.touchClaims(pendingIDs, requestClaims);
       if (!fetched) {
-        Logger(`No chunks were found for the following IDs: ${requestIDs.join(", ")}`);
-        for (const chunkID of requestIDs) {
+        Logger(`No chunks were found for the following IDs: ${pendingIDs.join(", ")}`);
+        for (const chunkID of pendingIDs) {
           this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
         }
         return;
       }
-      const chunks = fetched.filter((chunk) => isValidChunk2(chunk));
-      if (chunks.length !== fetched.length) {
-        Logger(
-          `Some fetched chunks are invalid and will be ignored: (${fetched.length - chunks.length} / ${fetched.length}).`,
-          LOG_LEVEL_VERBOSE
-        );
-        for (const chunk of fetched) {
-          if (!isValidChunk2(chunk)) {
-            Logger(`Invalid chunk: ${JSON.stringify(chunk)}`, LOG_LEVEL_VERBOSE);
-          }
-        }
-      }
-      if (chunks.length === 0) {
-        Logger(`No valid chunks were found for the following IDs: ${requestIDs.join(", ")}`);
-      }
-      const missingIDs = requestIDs.filter((id) => !chunks.some((chunk) => chunk._id === id));
+      const classification = classifyFetchedChunks(pendingIDs, fetched);
+      this.logFetchedChunks(pendingIDs, classification);
+      const { chunks, missingIds, invalidIds } = classification;
       try {
-        if (chunks.length === 0) {
-          return;
-        }
-        Logger(`Writing fetched chunks (${chunks.length}) to the database...`);
-        const result = await this.chunkManager.write(
-          chunks,
-          {
-            skipCache: true,
-            force: true
-            // Force writing to ensure the chunks with existing _rev.
-          },
-          "ChunkFetcher"
-        );
-        this.touchClaims(requestIDs);
-        if (result.result === true) {
-          Logger(`Fetched chunks were stored successfully: ${chunks.length}`, LOG_LEVEL_VERBOSE);
-        } else {
-          Logger(
-            `Fetched chunks could not be stored: ${chunks.map((chunk) => chunk._id).join(", ")}`,
-            LOG_LEVEL_VERBOSE
+        if (chunks.length > 0) {
+          Logger(`Writing fetched chunks (${chunks.length}) to the database...`);
+          const result = await this.chunkManager.write(
+            chunks,
+            {
+              skipCache: true,
+              force: true
+              // Force writing to ensure the chunks with existing _rev.
+            },
+            "ChunkFetcher"
           );
+          this.touchClaims(pendingIDs, requestClaims);
+          if (result.result === true) {
+            Logger(`Fetched chunks were stored successfully: ${chunks.length}`, LOG_LEVEL_VERBOSE);
+          } else {
+            Logger(
+              `Fetched chunks could not be stored: ${chunks.map((chunk) => chunk._id).join(", ")}`,
+              LOG_LEVEL_VERBOSE
+            );
+          }
         }
       } catch (error) {
         Logger(`An error occurred while storing fetched chunks!`, LOG_LEVEL_VERBOSE);
@@ -233,23 +381,24 @@ class ChunkFetcher {
       } finally {
         for (const chunk of chunks) {
           this.chunkManager.emitEvent(EVENT_CHUNK_FETCHED, chunk);
+          this.settleClaim(chunk._id, requestClaims.get(chunk._id));
         }
-        for (const chunkID of missingIDs) {
-          this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
+        for (const invalidID of this.getActiveClaimIds(invalidIds, requestClaims)) {
+          this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, invalidID);
+          this.settleClaim(invalidID, requestClaims.get(invalidID));
         }
       }
+      this.applyMissingChunkResults(missingIds, requestClaims, requestCompletionVersion);
     } catch (error) {
       Logger("An error occurred while fetching remote chunks.", LOG_LEVEL_VERBOSE);
       Logger(error, LOG_LEVEL_VERBOSE);
     } finally {
-      for (const id of requestIDs) {
-        this.settleClaim(id);
+      for (const [id, pending] of requestClaims) {
+        if (pending.inFlight) this.settleClaim(id, pending);
       }
       this.currentProcessing--;
       this.previousRequestTime = Date.now();
-      if (this.queue.length > 0) {
-        compatGlobal.setTimeout(() => void this.requestMissingChunks(), 0);
-      }
+      this.scheduleRequest();
     }
   }
 }

package/dist/managers/HashManager/HashManager.d.ts

@@ -9,6 +9,9 @@ export declare class HashManager extends HashManagerCore {
      * Instance of the hash manager currently in use.
      */
     manager: HashManagerCore;
+    private chunkIdGenerator?;
+    clearCaches(): void;
+    private computeIndependentChunkId;
     /**
      * Checks whether the specified hash algorithm is available.
      *

package/dist/managers/HashManager/HashManager.js

@@ -2,7 +2,7 @@ import { FallbackWasmHashManager, XXHash32RawHashManager, XXHash64HashManager }
 import { FallbackPureJSHashManager, PureJSHashManager, SHA1HashManager } from "./PureJSHashManager.js";
 import { HashEncryptedPrefix, HashManagerCore } from "./HashManagerCore.js";
 import { LOG_LEVEL_VERBOSE, Logger } from "../../common/logger.js";
-import { computeKeyedId, configuredIdKey } from "../../common/idDerivation.js";
+import { createChunkIdGenerator, configuredIdKey } from "../../common/idDerivation.js";
 const HashManagers = [
   XXHash64HashManager,
   XXHash32RawHashManager,
@@ -17,6 +17,22 @@ class HashManager extends HashManagerCore {
    * Instance of the hash manager currently in use.
    */
   manager = void 0;
+  chunkIdGenerator;
+  clearCaches() {
+    this.chunkIdGenerator = void 0;
+  }
+  async computeIndependentChunkId(key, piece) {
+    let cached = this.chunkIdGenerator;
+    if (cached?.key !== key) {
+      const entry = { key, task: createChunkIdGenerator(key) };
+      this.chunkIdGenerator = cached = entry;
+      void entry.task.catch(() => {
+        if (this.chunkIdGenerator === entry) this.clearCaches();
+      });
+    }
+    const generate = await cached.task;
+    return await generate(piece);
+  }
   /**
    * Checks whether the specified hash algorithm is available.
    *
@@ -75,13 +91,16 @@ class HashManager extends HashManagerCore {
     const settings = this.options.settingService.currentSettings();
     const key = configuredIdKey(settings);
     if (settings.encrypt && key) {
-      return HashEncryptedPrefix + await computeKeyedId(key, "chunk", piece);
+      return HashEncryptedPrefix + await this.computeIndependentChunkId(key, piece);
     }
-    return await this.manager.computeHash(piece);
+    this.clearCaches();
+    return settings.encrypt ? HashEncryptedPrefix + await this.manager.computeHashWithEncryption(piece) : await this.manager.computeHashWithoutEncryption(piece);
   }
   usesIndependentIdKey() {
     const settings = this.options.settingService.currentSettings();
-    return settings.encrypt && configuredIdKey(settings) !== false;
+    const active = settings.encrypt && configuredIdKey(settings) !== false;
+    if (!active) this.clearCaches();
+    return active;
   }
   /**
    * Computes the hash value without encryption.
@@ -100,7 +119,8 @@ class HashManager extends HashManagerCore {
    */
   computeHashWithEncryption(piece) {
     const key = configuredIdKey(this.options.settingService.currentSettings());
-    if (key) return computeKeyedId(key, "chunk", piece);
+    if (key) return this.computeIndependentChunkId(key, piece);
+    this.clearCaches();
     return this.manager.computeHashWithEncryption(piece);
   }
 }

package/dist/managers/LiveSyncManagers.js

@@ -39,6 +39,8 @@ class LiveSyncManagers {
   }
   async teardownManagers() {
     this.log("Teardown LiveSync Managers...", LOG_LEVEL_VERBOSE);
+    this.hashManager?.clearCaches();
+    this.entryManager?.hashManager?.clearCaches();
     if (this.changeManager) {
       this.changeManager.teardown();
       this.changeManager = void 0;
@@ -123,8 +125,12 @@ class LiveSyncManagers {
   }
   clearCaches() {
     this.chunkManager?.clearCaches();
+    this.hashManager?.clearCaches();
+    this.entryManager?.hashManager?.clearCaches();
   }
   async prepareHashFunction() {
+    this.hashManager?.clearCaches();
+    this.entryManager?.hashManager?.clearCaches();
     this.hashManager = new HashManager({
       settingService: this.options.settingService
     });