@remnic/core 10.0.93 → 10.0.94

package/dist/{chunk-NT7F7WJQ.js → chunk-RFUL2ZIQ.js}

@@ -9,15 +9,19 @@ import {
   SecureStoreDecryptError,
   SecureStoreLockedError
 } from "./chunk-EYGOM6AI.js";
+import {
+  isErrnoCode
+} from "./chunk-KNA3BXXL.js";
 import {
   log
 } from "./chunk-3WCASNQZ.js";
 
 // src/storage/buffer-drain-claims.ts
+import { randomUUID } from "crypto";
 import { hostname } from "os";
 import { z } from "zod";
 import { existsSync } from "fs";
-import { mkdir, readdir as readdir2, rename, unlink as unlink2, writeFile } from "fs/promises";
+import { mkdir, readdir as readdir2, readFile, rename, stat, unlink as unlink2, writeFile } from "fs/promises";
 import path2 from "path";
 
 // src/storage/buffer-session-files.ts
@@ -89,16 +93,16 @@ var BufferSessionFiles = class {
    */
   async assertDirSafe() {
     if (this.dirVerified) return;
-    let stat = null;
+    let stat2 = null;
     try {
-      stat = await lstat(this.dir);
+      stat2 = await lstat(this.dir);
     } catch (err) {
       if (!(err instanceof Error) || !isMissing(asFsError(err))) throw err;
     }
-    if (stat && (stat.isSymbolicLink() || !stat.isDirectory())) {
+    if (stat2 && (stat2.isSymbolicLink() || !stat2.isDirectory())) {
       throw new Error(`buffer session directory must be a real directory, not a symlink or file: ${this.dir}`);
     }
-    this.dirVerified = stat !== null;
+    this.dirVerified = stat2 !== null;
   }
   /** Persist ONE session atomically. Compact JSON: machine-read state, and
    * pretty-printing a large buffer costs CPU and bytes on every observe.
@@ -258,17 +262,19 @@ function turnCounts(turns) {
   }
   return counts;
 }
-function mergeTurns(existing, incoming) {
-  const merged = [...existing];
-  const existingCounts = turnCounts(existing);
+function appendNewOccurrences(target, counts, incoming) {
   const incomingCounts = /* @__PURE__ */ new Map();
+  const added = [];
   for (const turn of incoming) {
     const key = bufferTurnIdentityKey(turn);
     const count = (incomingCounts.get(key) ?? 0) + 1;
     incomingCounts.set(key, count);
-    if (count > (existingCounts.get(key) ?? 0)) merged.push(turn);
+    if (count > (counts.get(key) ?? 0)) {
+      target.push(turn);
+      added.push(key);
+    }
   }
-  return merged;
+  for (const key of added) counts.set(key, (counts.get(key) ?? 0) + 1);
 }
 function preserveSourceOccurrences(target, source, identity) {
   const targetCount = turnCounts(target).get(identity) ?? 0;
@@ -297,22 +303,28 @@ function removeOneByIdentity(turns, identity) {
 function takeByIdentity(source, identity, count) {
   return source.filter((turn) => bufferTurnIdentityKey(turn) === identity).slice(0, count);
 }
-function mergeEntries(claim, incoming) {
+function foldRecords(claim, records) {
+  const consumed = new Set(claim.foldedRecords);
+  const fresh = records.filter((record) => !consumed.has(record.id));
+  if (fresh.length === 0) return claim;
+  const turns = [...claim.entry.turns];
+  const retainedTurns = [...claim.entry.retainedTurns ?? []];
+  const activeCounts = turnCounts(turns);
+  const retainedCounts = turnCounts(retainedTurns);
   const entry = emptyEntry();
-  entry.turns = mergeTurns(
-    claim.entry.turns,
-    subtractBufferTurns(incoming.turns, claim.drained ?? [])
-  );
-  entry.retainedTurns = mergeTurns(
-    claim.entry.retainedTurns ?? [],
-    subtractBufferTurns(incoming.retainedTurns ?? [], claim.drainedRetained ?? [])
-  );
-  if (entry.retainedTurns.length === 0) delete entry.retainedTurns;
-  const stamps = [claim.entry.lastExtractionAt, incoming.lastExtractionAt].filter(
-    (value) => value !== null && value !== void 0
-  );
-  entry.lastExtractionAt = stamps.sort().at(-1) ?? null;
-  entry.extractionCount = Math.max(claim.entry.extractionCount, incoming.extractionCount);
+  entry.lastExtractionAt = claim.entry.lastExtractionAt;
+  entry.extractionCount = claim.entry.extractionCount;
+  for (const { entry: incoming } of fresh) {
+    appendNewOccurrences(turns, activeCounts, subtractBufferTurns(incoming.turns, claim.drained ?? []));
+    appendNewOccurrences(retainedTurns, retainedCounts, subtractBufferTurns(incoming.retainedTurns ?? [], claim.drainedRetained ?? []));
+    const stamps = [entry.lastExtractionAt, incoming.lastExtractionAt].filter(
+      (value) => value !== null && value !== void 0
+    );
+    entry.lastExtractionAt = stamps.sort().at(-1) ?? null;
+    entry.extractionCount = Math.max(entry.extractionCount, incoming.extractionCount);
+  }
+  entry.turns = turns;
+  if (retainedTurns.length > 0) entry.retainedTurns = retainedTurns;
   return { ...claim, entry };
 }
 function isOwnerAlive(lease) {
@@ -344,7 +356,8 @@ var claimBodySchema = z.object({
   claimedAt: z.string().optional(),
   drained: z.array(z.unknown()).optional(),
   drainedRetained: z.array(z.unknown()).optional(),
-  movedTurns: z.array(z.unknown()).optional()
+  movedTurns: z.array(z.unknown()).optional(),
+  foldedRecords: z.array(z.string()).optional()
 });
 function parseBody(raw) {
   let body;
@@ -376,8 +389,49 @@ function parseBody(raw) {
   if (Array.isArray(body.movedTurns)) {
     claim.movedTurns = body.movedTurns;
   }
+  if (body.foldedRecords) claim.foldedRecords = body.foldedRecords;
   return claim;
 }
+var inboxRecordSchema = z.object({
+  id: z.string().min(1),
+  turns: z.array(z.unknown()),
+  retainedTurns: z.array(z.unknown()).optional(),
+  lastExtractionAt: z.string().nullable().optional(),
+  extractionCount: z.number().optional()
+});
+function parseInbox(raw) {
+  const records = [];
+  for (const line of raw.split("\n")) {
+    if (line.length === 0) continue;
+    let record;
+    try {
+      record = inboxRecordSchema.parse(JSON.parse(line));
+    } catch {
+      continue;
+    }
+    const turns = record.turns;
+    const entry = {
+      turns,
+      lastExtractionAt: record.lastExtractionAt ?? null,
+      extractionCount: Number.isFinite(record.extractionCount) ? Number(record.extractionCount) : 0
+    };
+    if (record.retainedTurns) {
+      entry.retainedTurns = record.retainedTurns;
+    }
+    records.push({ id: record.id, entry });
+  }
+  return records;
+}
+function pendingTurns(entry) {
+  return entry.turns.length + (entry.retainedTurns?.length ?? 0);
+}
+var summarySchema = z.object({
+  key: z.string().min(1),
+  size: z.number(),
+  mtimeMs: z.number(),
+  pending: z.number(),
+  drained: z.number()
+});
 var leaseSchema = z.object({
   ownerId: z.string().min(1),
   pid: z.number().optional(),
@@ -405,6 +459,8 @@ function parseLease(raw) {
 function leaseBelongsToProcess(lease, ownerId) {
   return lease !== null && lease.ownerId === ownerId && lease.pid === process.pid && lease.host === hostname();
 }
+var INBOX_SUFFIX = ".inbox.jsonl";
+var COUNT_SUFFIX = ".count";
 var LIST_PAGE_SCAN_FACTOR = 8;
 var BufferDrainClaims = class {
   constructor(stateDir, io) {
@@ -425,6 +481,10 @@ var BufferDrainClaims = class {
   lockFileFor(key) {
     return `${this.fileFor(key)}.lock`;
   }
+  /** Append-only hand-in log next to the body (see the file header). */
+  inboxFor(key) {
+    return `${this.fileFor(key)}${INBOX_SUFFIX}`;
+  }
   /** Sync existence check for hot-path gates (`SmartBuffer.isSessionClaimed`). */
   isClaimed(key) {
     return existsSync(this.fileFor(key));
@@ -440,13 +500,100 @@ var BufferDrainClaims = class {
       return null;
     }
   }
-  /** The claim content for `key`, or null when the key is not claimed. */
-  load(key) {
-    return this.readClaimBody(key);
+  /**
+   * Every inbox record, oldest first. Only "no inbox" reads as empty: any
+   * other failure (a locked or undecryptable store) propagates, because a
+   * compaction that mistook an unreadable inbox for an empty one would delete
+   * the handed-in turns.
+   */
+  async readInbox(key) {
+    try {
+      return parseInbox(await this.io.read(this.inboxFor(key)));
+    } catch (err) {
+      if (isErrnoCode(err, "ENOENT")) return [];
+      throw err;
+    }
   }
-  async writeClaimBody(key, claim) {
+  /**
+   * The body with its inbox folded in, plus the ids of every record the inbox
+   * holds (what a compaction must list as consumed). The inbox is read BEFORE
+   * the body: a compaction landing between the two reads leaves records the
+   * newer body already lists, which the fold skips by id, whereas the reverse
+   * order could pair an old body with a deleted inbox and lose their turns.
+   */
+  async readFolded(key) {
+    const records = await this.readInbox(key);
+    const body = await this.readClaimBody(key);
+    if (!body) return null;
+    return { claim: foldRecords(body, records), consumed: records.map((record) => record.id) };
+  }
+  /** The claim content for `key` (body plus inbox), or null when it is not claimed. */
+  async load(key) {
+    return (await this.readFolded(key))?.claim ?? null;
+  }
+  /**
+   * Rewrite the body from a folded claim and consume the inbox it folded.
+   * Order is the crash contract: body first (it lists `consumed` so a replay
+   * skips them), then the summary, then the inbox delete. Callers hold the
+   * claim lock, so no append can land between the read and the delete.
+   */
+  async writeClaimBody(key, claim, consumed = []) {
     await mkdir(this.dir, { recursive: true });
-    await this.io.write(this.fileFor(key), JSON.stringify({ ...claim, key }));
+    const bodyPath = this.fileFor(key);
+    const foldedRecords = consumed.length > 0 ? [...consumed] : void 0;
+    await this.io.write(bodyPath, JSON.stringify({ ...claim, key, foldedRecords }));
+    await stat(bodyPath).then((identity) => this.writeSummary(bodyPath, claim, identity), () => {
+    });
+    await unlink2(this.inboxFor(key)).catch(() => {
+    });
+  }
+  /**
+   * Best-effort: a missing or stale summary only costs one body parse.
+   * `identity` MUST come from a stat taken before `claim` was read (or while
+   * the lock holds the body still), never after, or the counts could be
+   * tagged with a newer body's identity.
+   */
+  async writeSummary(bodyPath, claim, identity) {
+    const summary = {
+      key: claim.key,
+      size: identity.size,
+      mtimeMs: identity.mtimeMs,
+      pending: pendingTurns(claim.entry),
+      drained: (claim.drained?.length ?? 0) + (claim.drainedRetained?.length ?? 0)
+    };
+    try {
+      await writeFile(`${bodyPath}${COUNT_SUFFIX}`, JSON.stringify(summary));
+    } catch (err) {
+      log.debug(`buffer drain: claim summary write skipped for ${claim.key}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    return summary;
+  }
+  /**
+   * Turn counts of the BODY at `bodyPath` without parsing it: the summary file
+   * is trusted only while the body's size and mtime still match the ones it
+   * was written against; otherwise the body is parsed once and the summary
+   * refreshed. Null when the claim body is absent or unreadable.
+   */
+  async readSummary(bodyPath) {
+    let identity;
+    try {
+      identity = await stat(bodyPath);
+    } catch {
+      return null;
+    }
+    try {
+      const cached = summarySchema.parse(JSON.parse(await readFile(`${bodyPath}${COUNT_SUFFIX}`, "utf8")));
+      if (cached.size === identity.size && cached.mtimeMs === identity.mtimeMs) return cached;
+    } catch {
+    }
+    let claim;
+    try {
+      claim = parseBody(await this.io.read(bodyPath));
+    } catch (err) {
+      log.debug(`buffer drain: skipping unreadable claim file ${path2.basename(bodyPath)}: ${err instanceof Error ? err.message : String(err)}`);
+      return null;
+    }
+    return claim ? await this.writeSummary(bodyPath, claim, identity) : null;
   }
   async readLease(key) {
     try {
@@ -487,6 +634,8 @@ var BufferDrainClaims = class {
       async (acquired) => {
         if (!acquired) return "already_claimed";
         if (this.isClaimed(key)) return "already_claimed";
+        await unlink2(this.inboxFor(key)).catch(() => {
+        });
         try {
           await rename(sessionPath, this.fileFor(key));
         } catch (err) {
@@ -497,11 +646,13 @@ var BufferDrainClaims = class {
           throw err;
         }
         const claim = await this.readClaimBody(key);
-        if (!claim || claim.entry.turns.length === 0 && (claim.entry.retainedTurns?.length ?? 0) === 0) {
+        if (!claim || pendingTurns(claim.entry) === 0) {
           await unlink2(this.fileFor(key)).catch(() => {
           });
           return "empty";
         }
+        await stat(this.fileFor(key)).then((identity) => this.writeSummary(this.fileFor(key), claim, identity), () => {
+        });
         await this.writeLease(key, lease);
         return "claimed";
       }
@@ -545,20 +696,21 @@ var BufferDrainClaims = class {
     );
   }
   /**
-   * Merge a live owner's in-memory entry INTO the claim (the handover path a
-   * buffer owner takes instead of persisting a claimed session). Serialized
-   * against drainer progress writes on the same lock.
+   * Hand a live owner's in-memory entry to the claim (the path a buffer owner
+   * takes instead of persisting a claimed session): ONE append to the inbox,
+   * O(incoming), never a read or rewrite of the claim body. Readers fold the
+   * inbox over the body with the identity-aware merge and the drainer's next
+   * progress write compacts it. The lock serializes the append against that
+   * compaction, so a record can never land between its read and its delete.
    */
   async mergeIntoClaim(key, incoming) {
     await withHeldFileLock(this.lockFileFor(key), { staleMs: CLAIM_LOCK_STALE_MS, maxWaitMs: CLAIM_LOCK_MAX_WAIT_MS }, async (acquired) => {
       if (!acquired) {
         throw new Error(`buffer drain claim for ${key} is busy; retrying the save later`);
       }
-      const claim = await this.readClaimBody(key);
-      if (!claim) {
-        return;
-      }
-      await this.writeClaimBody(key, mergeEntries(claim, incoming));
+      if (!this.isClaimed(key)) return;
+      await this.io.append(this.inboxFor(key), `
+${JSON.stringify({ id: randomUUID(), key, ...incoming })}`);
     });
   }
   /** Drainer progress: replace the claim content with what is left to do.
@@ -582,8 +734,9 @@ var BufferDrainClaims = class {
       if (!leaseBelongsToProcess(lease, ownerId)) {
         throw new Error(`buffer drain claim for ${key} is no longer owned by ${ownerId}; progress write skipped`);
       }
-      const claim = await this.readClaimBody(key);
-      if (!claim) return;
+      const folded = await this.readFolded(key);
+      if (!folded) return;
+      const { claim, consumed } = folded;
       const seenTurns = seen?.turns ?? claim.entry.turns;
       const seenRetained = seen?.retainedTurns ?? claim.entry.retainedTurns ?? [];
       const entryRetained = entry.retainedTurns ?? [];
@@ -660,7 +813,7 @@ var BufferDrainClaims = class {
         drained,
         drainedRetained,
         movedTurns: moved.length > 0 ? moved : void 0
-      });
+      }, consumed);
     });
   }
   /** Extend the lease. "lost" means another drainer took the claim over.
@@ -702,13 +855,14 @@ var BufferDrainClaims = class {
         if (!acquired) return "busy";
         const lease = await this.readLease(key);
         if (!leaseBelongsToProcess(lease, ownerId)) return "busy";
-        const claim = await this.readClaimBody(key);
-        if (!claim) {
+        const folded = await this.readFolded(key);
+        if (!folded) {
           await unlink2(this.leaseFileFor(key)).catch(() => {
           });
           return "acked";
         }
-        if (claim.entry.turns.length > 0 || (claim.entry.retainedTurns?.length ?? 0) > 0) return "not_empty";
+        if (folded.consumed.length > 0) await this.writeClaimBody(key, folded.claim, folded.consumed);
+        if (pendingTurns(folded.claim.entry) > 0) return "not_empty";
         await unlink2(this.leaseFileFor(key)).catch(() => {
         });
         return "acked";
@@ -744,39 +898,60 @@ var BufferDrainClaims = class {
    * merge into it, never release it.
    */
   async isDrained(key) {
-    const claim = await this.readClaimBody(key);
-    if (!claim) return false;
-    if ((claim.drained?.length ?? 0) === 0 && (claim.drainedRetained?.length ?? 0) === 0) return false;
-    if (claim.entry.turns.length > 0 || (claim.entry.retainedTurns?.length ?? 0) > 0) return false;
-    return await this.readLease(key) === null;
+    if (await this.readLease(key) !== null) return false;
+    const summary = await this.readSummary(this.fileFor(key));
+    if (!summary || summary.drained === 0 || summary.pending > 0) return false;
+    const claim = await this.load(key);
+    return claim !== null && (claim.drained?.length ?? 0) + (claim.drainedRetained?.length ?? 0) > 0 && pendingTurns(claim.entry) === 0;
   }
-  /** Drop the claim file entirely (its owner consumed the tombstone). */
+  /** Drop the claim entirely (its owner consumed the tombstone), inbox and summary included. */
   async release(key) {
-    await unlink2(this.fileFor(key)).catch(() => {
-    });
-    await unlink2(this.leaseFileFor(key)).catch(() => {
-    });
+    const bodyPath = this.fileFor(key);
+    for (const file of [this.inboxFor(key), bodyPath, `${bodyPath}${COUNT_SUFFIX}`, this.leaseFileFor(key)]) {
+      await unlink2(file).catch(() => {
+      });
+    }
   }
-  /** Every claim with its lease state (CLI scan + summaries). */
-  async list() {
-    let names;
+  /** Claim body file names in the drain directory (no leases, inboxes, summaries or locks). */
+  async claimNames() {
     try {
-      names = (await readdir2(this.dir)).filter((name) => name.endsWith(".json") && !name.endsWith(".lease.json"));
+      return (await readdir2(this.dir)).filter((name) => name.endsWith(".json") && !name.endsWith(".lease.json")).sort();
     } catch (err) {
-      if (err?.code === "ENOENT") return [];
+      if (err?.code === "ENOENT") return null;
       throw err;
     }
+  }
+  /**
+   * One claim's listing from its summary plus its inbox: the body is never
+   * parsed for a count. While an inbox is unfolded the turn count is an UPPER
+   * bound (records can repeat turns the body holds or already drained); the
+   * drain that picks the claim up folds exactly and its ack compacts the
+   * inbox, so an over-count costs one cheap drain run, never a lost turn.
+   */
+  async describeClaim(name) {
+    const summary = await this.readSummary(path2.join(this.dir, name));
+    if (!summary) return null;
+    let inboxTurns = 0;
+    try {
+      for (const record of await this.readInbox(summary.key)) inboxTurns += pendingTurns(record.entry);
+    } catch (err) {
+      log.debug(`buffer drain: skipping claim ${name} with an unreadable inbox: ${err instanceof Error ? err.message : String(err)}`);
+      return null;
+    }
+    const lease = await this.readLease(summary.key);
+    return {
+      key: summary.key,
+      turnCount: summary.pending + inboxTurns,
+      leased: lease !== null,
+      leaseExpired: lease !== null ? isLeaseExpired(lease) : true
+    };
+  }
+  /** Every claim with its lease state (CLI scan + summaries). */
+  async list() {
     const out = [];
-    for (const name of names) {
-      const claim = await this.readClaimBodyByKey(name);
-      if (!claim) continue;
-      const lease = await this.readLease(claim.key);
-      out.push({
-        key: claim.key,
-        turnCount: claim.entry.turns.length + (claim.entry.retainedTurns?.length ?? 0),
-        leased: lease !== null,
-        leaseExpired: lease !== null ? isLeaseExpired(lease) : true
-      });
+    for (const name of await this.claimNames() ?? []) {
+      const listing = await this.describeClaim(name);
+      if (listing) out.push(listing);
     }
     return out;
   }
@@ -786,20 +961,14 @@ var BufferDrainClaims = class {
    * `nextCursor` by a previous page — file-name order is stable, so a
    * rotating cursor visits every claim fairly without ever materializing
    * the whole directory's bodies). Empty tombstones never count toward the
-   * limit; the number of bodies read per call is bounded separately
+   * limit; the number of claims examined per call is bounded separately
    * (`LIST_PAGE_SCAN_FACTOR` per requested entry), and the cursor advances
    * past every scanned name so a long tombstone run is crossed over several
    * pages instead of starving the pending claims behind it.
    */
   async listPage(afterName, limit) {
-    let names;
-    try {
-      names = (await readdir2(this.dir)).filter((name) => name.endsWith(".json") && !name.endsWith(".lease.json")).sort();
-    } catch (err) {
-      const errno = err?.code;
-      if (errno === "ENOENT") return { entries: [] };
-      throw err;
-    }
+    const names = await this.claimNames();
+    if (names === null) return { entries: [] };
     const start = afterName === void 0 ? 0 : names.findIndex((name) => name > afterName);
     const out = [];
     let cursor;
@@ -810,21 +979,50 @@ var BufferDrainClaims = class {
       const name = names[index];
       scanned += 1;
       cursor = name;
-      const claim = await this.readClaimBodyByKey(name);
-      if (!claim) continue;
-      const turnCount = claim.entry.turns.length + (claim.entry.retainedTurns?.length ?? 0);
-      if (turnCount <= 0) continue;
-      const lease = await this.readLease(claim.key);
-      out.push({
-        key: claim.key,
-        turnCount,
-        leased: lease !== null,
-        leaseExpired: lease !== null ? isLeaseExpired(lease) : true
-      });
+      const listing = await this.describeClaim(name);
+      if (listing && listing.turnCount > 0) out.push(listing);
     }
     return { entries: out, nextCursor: cursor };
   }
-  async readClaimBodyByKey(name) {
+  /**
+   * Rollback helper: fold every claim's inbox into its body and delete the
+   * inboxes, so a build that predates the inbox (it never reads one) sees every
+   * handed-in turn. Run it with the daemon stopped, BEFORE downgrading; the
+   * same fold is what compaction does, so it is safe to run on a live store.
+   * Returns how many inboxes were folded.
+   */
+  async foldAllInboxes() {
+    let names;
+    try {
+      names = (await readdir2(this.dir)).filter((name) => name.endsWith(INBOX_SUFFIX));
+    } catch (err) {
+      if (isErrnoCode(err, "ENOENT")) return 0;
+      throw err;
+    }
+    let folded = 0;
+    for (const name of names) {
+      const claim = await this.readClaimBodyByName(name.slice(0, -INBOX_SUFFIX.length));
+      if (!claim) continue;
+      folded += await withHeldFileLock(
+        this.lockFileFor(claim.key),
+        { staleMs: CLAIM_LOCK_STALE_MS, maxWaitMs: CLAIM_LOCK_MAX_WAIT_MS },
+        async (acquired) => {
+          if (!acquired) throw new Error(`buffer drain claim for ${claim.key} is busy; its inbox was not folded`);
+          const current = await this.readFolded(claim.key);
+          if (!current) return 0;
+          if (current.consumed.length === 0) {
+            await unlink2(this.inboxFor(claim.key)).catch(() => {
+            });
+            return 0;
+          }
+          await this.writeClaimBody(claim.key, current.claim, current.consumed);
+          return 1;
+        }
+      );
+    }
+    return folded;
+  }
+  async readClaimBodyByName(name) {
     try {
       return parseBody(await this.io.read(path2.join(this.dir, name)));
     } catch (err) {
@@ -840,4 +1038,4 @@ export {
   legacyStateToEntries,
   BufferSessionFiles
 };
-//# sourceMappingURL=chunk-NT7F7WJQ.js.map
+//# sourceMappingURL=chunk-RFUL2ZIQ.js.map