@voybio/ace-swarm 2.4.3 → 2.4.6

package/dist/store/ace-packed-store.js

@@ -2,9 +2,9 @@
  * AcePackedStore — ACEPACK v2
  *
  * Single-file binary store. One file, no subdirectories, no companion files.
- * Events grow in the columnar section indefinitely (Option A). compact() only
- * reclaims dead space in the KV chunk region — it does NOT reset or archive
- * events. Historical event replay across all past runs is always available.
+ * Recent events live in the hot columnar section. Older contiguous event
+ * prefixes can be moved into BGZF-compressed KV archive batches with
+ * archiveColdEvents(); getEntries() reads both layers transparently.
  *
  * File layout:
  *
@@ -45,9 +45,9 @@
  * Timestamp/kind scanning reads only the fixed-width columns, never the pool.
  *
  * compact() removes dead KV space (overwritten keys) via atomic tmp-rename.
- * Events are preserved across compact() — they accumulate for the workspace
- * lifetime. See HOT_COLD_EVENT_TIERING.md for a future branch that adds
- * BGZF batch archiving for very long-lived workspaces.
+ * Events are preserved across compact(). For very long-lived workspaces,
+ * archiveColdEvents() moves old event prefixes into archive/events/* BGZF
+ * batches and advances evtBaseId.
  *
  * Backward compat: v1 files (64-byte header) are migrated on first commit().
  * v1 events live in core/log/ KV blobs and are pulled into the columnar section.
@@ -56,7 +56,16 @@ import { readFileSync, writeFileSync, existsSync, mkdirSync, renameSync } from "
 import { open as fsOpen } from "node:fs/promises";
 import { dirname } from "node:path";
 import { MAGIC, STORE_VERSION, HEADER_SIZE, HEADER_SIZE_V1, ContentSource, EntityKind, } from "./types.js";
-import { invalidateStoreSnapshotCache } from "./store-snapshot.js";
+import { invalidateStoreSnapshotCache, validateChunkBounds } from "./store-snapshot.js";
+import { crc32 } from "./crc32.js";
+import { bgzfCompress, bgzfDecompress } from "./bgzf.js";
+// ── Corruption error ─────────────────────────────────────────────────────────
+export class AcePackedStoreCorruptionError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "AcePackedStoreCorruptionError";
+    }
+}
 // ── Constants ─────────────────────────────────────────────────────────────────
 // Header field byte offsets (big-endian throughout)
 const H_KV_IDX_OFF = 16; // uint64: KV index offset
@@ -118,6 +127,9 @@ function readHeaderV2(buf) {
         evtBaseId: isV2 ? readU32(buf, H_EVT_BASE) : 0,
     };
 }
+const ARCHIVE_INDEX_KEY = "archive/events/__index";
+const DEFAULT_ARCHIVE_OLDER_THAN_MS = 7 * 24 * 60 * 60 * 1000;
+const DEFAULT_ARCHIVE_BATCH_SIZE = 5_000;
 // ── Columnar event section serialiser ────────────────────────────────────────
 function serializeEventSection(events) {
     const N = events.length;
@@ -162,6 +174,13 @@ function serializeEventSection(events) {
     }
     return buf;
 }
+/** Minimum byte length an event section header must have to hold N events. */
+function minEventSectionLength(N) {
+    const pad = (4 - (N % 4)) % 4;
+    // [4] count + [8N] ts + [N] kind + [N] source + [N] flags + [pad]
+    // + [4N] pay_off + [4N] pay_len + [4] pool_len  (payload pool bytes checked per-record below)
+    return 4 + 8 * N + N + N + N + pad + 4 * N + 4 * N + 4;
+}
 function deserializeEventSection(buf) {
     if (buf.length < 4)
         return [];
@@ -169,6 +188,10 @@ function deserializeEventSection(buf) {
     const N = dv.getUint32(0, false);
     if (N === 0)
         return [];
+    const minLength = minEventSectionLength(N);
+    if (N < 0 || !Number.isFinite(N) || buf.length < minLength) {
+        throw new AcePackedStoreCorruptionError(`AcePackedStore: event section corrupt — count=${N} requires >= ${minLength} bytes, got ${buf.length}`);
+    }
     let pos = 4;
     const ts = [];
     for (let i = 0; i < N; i++) {
@@ -195,8 +218,17 @@ function deserializeEventSection(buf) {
         payLens.push(dv.getUint32(pos, false));
         pos += 4;
     }
-    pos += 4; // pool_length (implicit from payLens sum)
+    const poolLength = dv.getUint32(pos, false);
+    pos += 4;
     const poolStart = pos;
+    if (buf.length < poolStart + poolLength) {
+        throw new AcePackedStoreCorruptionError(`AcePackedStore: event section pool truncated — declared pool_length=${poolLength}, available=${buf.length - poolStart}`);
+    }
+    for (let i = 0; i < N; i++) {
+        if (payOffs[i] < 0 || payLens[i] < 0 || payOffs[i] + payLens[i] > poolLength) {
+            throw new AcePackedStoreCorruptionError(`AcePackedStore: event section record ${i} payload exceeds pool bounds (offset=${payOffs[i]}, length=${payLens[i]}, pool=${poolLength})`);
+        }
+    }
     return Array.from({ length: N }, (_, i) => ({
         ts: ts[i],
         kind: kinds[i],
@@ -209,12 +241,20 @@ function readKvBlobFromBytes(file, index, key) {
     const entry = index.get(key);
     if (!entry)
         return undefined;
+    if (!validateChunkBounds(entry, file.length)) {
+        throw new AcePackedStoreCorruptionError(`AcePackedStore: KV entry for '${key}' exceeds file bounds (offset=${entry.offset}, length=${entry.length}, fileSize=${file.length})`);
+    }
+    const prefixed = readU32(file, entry.offset);
+    if (prefixed !== entry.length) {
+        throw new AcePackedStoreCorruptionError(`AcePackedStore: KV entry for '${key}' length-prefix mismatch (index=${entry.length}, stored=${prefixed})`);
+    }
     const start = entry.offset + 4;
     const end = start + entry.length;
-    if (start < 0 || end > file.length) {
-        throw new Error(`AcePackedStore: KV entry for '${key}' exceeds file bounds`);
+    const chunk = file.slice(start, end);
+    if (typeof entry.crc === "number" && crc32(chunk) !== entry.crc) {
+        throw new AcePackedStoreCorruptionError(`AcePackedStore: KV entry for '${key}' CRC mismatch (expected=${entry.crc}, actual=${crc32(chunk)})`);
     }
-    return file.slice(start, end);
+    return chunk;
 }
 function loadCommittedStateFromBytes(file) {
     if (file.length < HEADER_SIZE_V1)
@@ -229,6 +269,7 @@ function loadCommittedStateFromBytes(file) {
             .filter((key) => key.startsWith("core/log/") && key !== "core/log/__seq")
             .sort();
         const committed = [];
+        let skippedMalformed = 0;
         for (const key of logKeys) {
             const entry = readKvBlobFromBytes(file, kvIndex, key);
             if (!entry)
@@ -244,9 +285,13 @@ function loadCommittedStateFromBytes(file) {
                 });
             }
             catch {
-                // Skip malformed legacy events during migration.
+                // Skip malformed legacy events during migration — availability over strictness, but count it.
+                skippedMalformed += 1;
             }
         }
+        if (skippedMalformed > 0) {
+            console.error(JSON.stringify({ event: "store_v1_migration_skipped_malformed", count: skippedMalformed }));
+        }
         for (const key of [...kvIndex.keys()]) {
             if (key.startsWith("core/log/") || key === "meta/log_entry_seq") {
                 kvIndex.delete(key);
@@ -257,6 +302,7 @@ function loadCommittedStateFromBytes(file) {
             kvChunkEnd: hdr.kvChunkEnd,
             committed,
             evtBaseId: 0,
+            skippedMalformed,
         };
     }
     return {
@@ -266,15 +312,63 @@ function loadCommittedStateFromBytes(file) {
             ? deserializeEventSection(file.slice(hdr.evtOffset, hdr.evtOffset + hdr.evtLength))
             : [],
         evtBaseId: hdr.evtBaseId,
+        skippedMalformed: 0,
+    };
+}
+function quarantineCorruptSource(storePath, file, operation, error) {
+    const quarantinePath = `${storePath}.corrupt-${Date.now()}`;
+    writeFileSync(quarantinePath, file);
+    const message = error instanceof Error ? error.message : String(error);
+    throw new AcePackedStoreCorruptionError(`AcePackedStore ${operation} refused: corrupt source store quarantined at ${quarantinePath}; original left in place. Run 'ace repair' or 'ace fsck --repair'. Cause: ${message}`);
+}
+function loadCommittedStateOrQuarantine(storePath, file, operation) {
+    try {
+        return loadCommittedStateFromBytes(file);
+    }
+    catch (error) {
+        quarantineCorruptSource(storePath, file, operation, error);
+    }
+}
+function eventRecordToEntry(rec, id) {
+    let parsed;
+    try {
+        parsed = JSON.parse(rec.blob);
+    }
+    catch {
+        return undefined;
+    }
+    return {
+        id,
+        ts: rec.ts,
+        kind: rec.kind,
+        content_source: rec.source,
+        key: parsed.key,
+        payload: parsed.payload,
+        parent_id: parsed.parent_id,
+        flags: rec.flags,
     };
 }
+function entryMatchesFilter(entry, filter) {
+    if (filter?.kind !== undefined && entry.kind !== filter.kind)
+        return false;
+    if (filter?.content_source !== undefined && entry.content_source !== filter.content_source)
+        return false;
+    if (filter?.keyPrefix && !entry.key.startsWith(filter.keyPrefix))
+        return false;
+    if (filter?.since !== undefined && entry.ts < filter.since)
+        return false;
+    if (!filter?.includeDeleted && (entry.flags ?? 0) & 0x01)
+        return false;
+    return true;
+}
 // ── File I/O helpers ──────────────────────────────────────────────────────────
-async function readExact(fh, buf, position) {
+async function readExact(fh, buf, position, context = {}) {
     let off = 0;
     while (off < buf.length) {
         const { bytesRead } = await fh.read(buf, off, buf.length - off, position + off);
-        if (bytesRead <= 0)
-            throw new Error("AcePackedStore: unexpected EOF");
+        if (bytesRead <= 0) {
+            throw new Error(`AcePackedStore: unexpected EOF reading key=${context.key ?? "<unknown>"} at offset=${position + off}, wanted=${buf.length - off} more bytes, got 0 (total wanted=${buf.length}, read so far=${off})`);
+        }
         off += bytesRead;
     }
 }
@@ -293,6 +387,7 @@ export class AcePackedStore {
     committed = []; // loaded from file
     pending = []; // written since last commit
     evtBaseId = 0; // events [0..evtBaseId] are in the archive
+    skippedMalformedCount = 0; // events dropped as malformed since open() — observable data loss
     // ── Lifecycle ───────────────────────────────────────────────────────────────
     async open(path, opts = {}) {
         this.storePath = path;
@@ -345,6 +440,7 @@ export class AcePackedStore {
         this.kvChunkEnd = loaded.kvChunkEnd;
         this.committed = loaded.committed;
         this.evtBaseId = loaded.evtBaseId;
+        this.skippedMalformedCount += loaded.skippedMalformed;
         this.pending = [];
         this.pendingKv.clear();
         this.pendingDeletes.clear();
@@ -354,7 +450,8 @@ export class AcePackedStore {
         if (this.readOnly || !this.fh || !this.storePath || !this.dirty)
             return;
         const currentFile = readFileSync(this.storePath);
-        const latest = loadCommittedStateFromBytes(new Uint8Array(currentFile.buffer, currentFile.byteOffset, currentFile.byteLength));
+        const latest = loadCommittedStateOrQuarantine(this.storePath, new Uint8Array(currentFile.buffer, currentFile.byteOffset, currentFile.byteLength), "commit");
+        this.skippedMalformedCount += latest.skippedMalformed;
         const nextCommitted = [...latest.committed, ...this.pending];
         const evtBytes = serializeEventSection(nextCommitted);
         const nextKvIndex = new Map();
@@ -363,17 +460,23 @@ export class AcePackedStore {
         for (const [key] of latest.kvIndex) {
             if (this.pendingDeletes.has(key) || this.pendingKv.has(key))
                 continue;
-            const value = readKvBlobFromBytes(currentFile, latest.kvIndex, key);
+            let value;
+            try {
+                value = readKvBlobFromBytes(currentFile, latest.kvIndex, key);
+            }
+            catch (error) {
+                quarantineCorruptSource(this.storePath, currentFile, "commit", error);
+            }
             if (!value)
                 continue;
             liveChunks.push({ value });
-            nextKvIndex.set(key, { offset: writeOff, length: value.length });
+            nextKvIndex.set(key, { offset: writeOff, length: value.length, crc: crc32(value) });
             writeOff += 4 + value.length;
         }
         for (const [key, value] of this.pendingKv) {
             const blob = value.slice();
             liveChunks.push({ value: blob });
-            nextKvIndex.set(key, { offset: writeOff, length: blob.length });
+            nextKvIndex.set(key, { offset: writeOff, length: blob.length, crc: crc32(blob) });
             writeOff += 4 + blob.length;
         }
         const idxJson = JSON.stringify(Object.fromEntries(nextKvIndex));
@@ -421,9 +524,8 @@ export class AcePackedStore {
     }
     /**
      * Compacts the KV chunk region — removes dead space left by overwritten keys.
-     * Events are NOT reset or archived; they accumulate in the columnar section
-     * for the workspace lifetime (Option A — grow forever).
-     * See HOT_COLD_EVENT_TIERING.md for the future branch that adds BGZF archiving.
+     * Events are preserved exactly as currently tiered. Use archiveColdEvents()
+     * first when old hot events should move into BGZF archive batches.
      */
     async compact() {
         if (!this.storePath || this.readOnly)
@@ -433,16 +535,22 @@ export class AcePackedStore {
         }
         // Rebuild KV chunk region with only live keys (removes dead space)
         const srcFile = readFileSync(this.storePath);
-        const latest = loadCommittedStateFromBytes(new Uint8Array(srcFile.buffer, srcFile.byteOffset, srcFile.byteLength));
+        const latest = loadCommittedStateOrQuarantine(this.storePath, new Uint8Array(srcFile.buffer, srcFile.byteOffset, srcFile.byteLength), "compact");
         const liveKv = new Map();
         const chunks = [];
         let writeOff = HEADER_SIZE;
         for (const [key] of latest.kvIndex) {
-            const chunk = readKvBlobFromBytes(srcFile, latest.kvIndex, key);
+            let chunk;
+            try {
+                chunk = readKvBlobFromBytes(srcFile, latest.kvIndex, key);
+            }
+            catch (error) {
+                quarantineCorruptSource(this.storePath, srcFile, "compact", error);
+            }
             if (!chunk)
                 continue;
             chunks.push(chunk);
-            liveKv.set(key, { offset: writeOff, length: chunk.length });
+            liveKv.set(key, { offset: writeOff, length: chunk.length, crc: crc32(chunk) });
             writeOff += 4 + chunk.length;
         }
         this.kvIndex = liveKv;
@@ -492,13 +600,30 @@ export class AcePackedStore {
         this.dirty = false;
     }
     async close() {
-        if (this.dirty) {
-            await this.commit();
+        let commitError;
+        try {
+            if (this.dirty) {
+                await this.commit();
+            }
         }
-        if (this.fh) {
-            await this.fh.close();
-            this.fh = null;
+        catch (error) {
+            commitError = error;
         }
+        finally {
+            // A corrupt-source commit can fail before commit() closes the handle. Do
+            // not rely on FileHandle garbage collection: Node warns that this becomes
+            // an error in a future release, and long-lived processes would leak fds.
+            if (this.fh) {
+                try {
+                    await this.fh.close();
+                }
+                finally {
+                    this.fh = null;
+                }
+            }
+        }
+        if (commitError !== undefined)
+            throw commitError;
     }
     // ── Core KV ─────────────────────────────────────────────────────────────────
     async get(key) {
@@ -511,10 +636,21 @@ export class AcePackedStore {
         const entry = this.kvIndex.get(key);
         if (!entry || !this.fh)
             return undefined;
+        const stats = await this.fh.stat();
+        if (!validateChunkBounds(entry, stats.size)) {
+            throw new AcePackedStoreCorruptionError(`AcePackedStore: KV entry for '${key}' exceeds file bounds (offset=${entry.offset}, length=${entry.length}, fileSize=${stats.size})`);
+        }
         const buf = new Uint8Array(entry.length);
-        await readExact(this.fh, buf, entry.offset + 4);
+        await readExact(this.fh, buf, entry.offset + 4, { key });
+        if (typeof entry.crc === "number" && crc32(buf) !== entry.crc) {
+            throw new AcePackedStoreCorruptionError(`AcePackedStore: KV entry for '${key}' CRC mismatch (expected=${entry.crc}, actual=${crc32(buf)})`);
+        }
         return buf;
     }
+    /** Events dropped as malformed since this instance was opened (v1 migration only). */
+    get skippedMalformedEventCount() {
+        return this.skippedMalformedCount;
+    }
     async set(key, value) {
         if (this.readOnly)
             throw new Error("AcePackedStore: read-only");
@@ -579,38 +715,169 @@ export class AcePackedStore {
         this.dirty = true;
         return { id, ts, kind: entry.kind, content_source: entry.content_source, key: entry.key, payload: entry.payload, flags: entry.flags };
     }
-    async getEntries(filter) {
-        const all = [...this.committed, ...this.pending];
+    async readArchiveIndex() {
+        return (await this.getJSON(ARCHIVE_INDEX_KEY)) ?? { version: 1, batches: [] };
+    }
+    async getArchivedEntries(filter) {
+        const index = await this.readArchiveIndex();
         const results = [];
-        let id = this.evtBaseId + 1;
-        for (const rec of all) {
-            let parsed;
-            try {
-                parsed = JSON.parse(rec.blob);
-            }
-            catch {
-                id++;
+        for (const batch of index.batches) {
+            if (filter?.since !== undefined && batch.ts_end < filter.since)
                 continue;
-            }
-            const entry = {
-                id: id++,
-                ts: rec.ts,
-                kind: rec.kind,
-                content_source: rec.source,
-                key: parsed.key,
-                payload: parsed.payload,
-                parent_id: parsed.parent_id,
-                flags: rec.flags,
-            };
-            if (filter?.kind !== undefined && entry.kind !== filter.kind)
+            const compressed = await this.get(batch.kv_key);
+            if (!compressed)
                 continue;
-            if (filter?.content_source !== undefined && entry.content_source !== filter.content_source)
+            const events = deserializeEventSection(bgzfDecompress(compressed));
+            let id = batch.evt_start;
+            for (const rec of events) {
+                const entry = eventRecordToEntry(rec, id++);
+                if (!entry) {
+                    this.skippedMalformedCount += 1;
+                    console.error(JSON.stringify({ event: "store_archive_entry_skipped_malformed", batch_id: batch.batch_id, total: this.skippedMalformedCount }));
+                    continue;
+                }
+                if (entryMatchesFilter(entry, filter))
+                    results.push(entry);
+            }
+        }
+        return results;
+    }
+    async archiveColdEvents(options = {}) {
+        if (!this.storePath || this.readOnly) {
+            return { batches_written: 0, events_archived: 0, bytes_written: 0 };
+        }
+        if (this.dirty) {
+            await this.commit();
+        }
+        const now = options.now ?? Date.now();
+        const olderThanMs = options.olderThanMs ?? DEFAULT_ARCHIVE_OLDER_THAN_MS;
+        const maxEventsPerBatch = Math.max(1, options.maxEventsPerBatch ?? DEFAULT_ARCHIVE_BATCH_SIZE);
+        const cutoff = now - olderThanMs;
+        const srcFile = readFileSync(this.storePath);
+        const latest = loadCommittedStateOrQuarantine(this.storePath, new Uint8Array(srcFile.buffer, srcFile.byteOffset, srcFile.byteLength), "archiveColdEvents");
+        let archiveCount = 0;
+        while (archiveCount < latest.committed.length && latest.committed[archiveCount]?.ts < cutoff) {
+            archiveCount += 1;
+        }
+        if (archiveCount === 0) {
+            return { batches_written: 0, events_archived: 0, bytes_written: 0 };
+        }
+        const existingIndexRaw = readKvBlobFromBytes(srcFile, latest.kvIndex, ARCHIVE_INDEX_KEY);
+        const existingIndex = existingIndexRaw
+            ? JSON.parse(DECODE.decode(existingIndexRaw))
+            : { version: 1, batches: [] };
+        const newChunks = new Map();
+        const newBatches = [];
+        let bytesWritten = 0;
+        for (let start = 0; start < archiveCount; start += maxEventsPerBatch) {
+            const batchEvents = latest.committed.slice(start, Math.min(start + maxEventsPerBatch, archiveCount));
+            const batchSeq = existingIndex.batches.length + newBatches.length + 1;
+            const batchId = `batch-${String(batchSeq).padStart(6, "0")}`;
+            const kvKey = `archive/events/${batchId}`;
+            const raw = serializeEventSection(batchEvents);
+            const compressed = bgzfCompress(raw);
+            newChunks.set(kvKey, compressed);
+            bytesWritten += compressed.length;
+            newBatches.push({
+                batch_id: batchId,
+                kv_key: kvKey,
+                evt_start: latest.evtBaseId + start + 1,
+                evt_end: latest.evtBaseId + start + batchEvents.length,
+                ts_start: batchEvents[0]?.ts ?? 0,
+                ts_end: batchEvents.at(-1)?.ts ?? 0,
+                count: batchEvents.length,
+                compressed_length: compressed.length,
+                uncompressed_length: raw.length,
+            });
+        }
+        const nextArchiveIndex = {
+            version: 1,
+            batches: [...existingIndex.batches, ...newBatches],
+        };
+        newChunks.set(ARCHIVE_INDEX_KEY, TEXT.encode(JSON.stringify(nextArchiveIndex)));
+        const liveKv = new Map();
+        const chunks = [];
+        let writeOff = HEADER_SIZE;
+        for (const [key] of latest.kvIndex) {
+            if (newChunks.has(key))
                 continue;
-            if (filter?.keyPrefix && !entry.key.startsWith(filter.keyPrefix))
+            let chunk;
+            try {
+                chunk = readKvBlobFromBytes(srcFile, latest.kvIndex, key);
+            }
+            catch (error) {
+                quarantineCorruptSource(this.storePath, srcFile, "archiveColdEvents", error);
+            }
+            if (!chunk)
                 continue;
-            if (filter?.since !== undefined && entry.ts < filter.since)
+            chunks.push({ value: chunk });
+            liveKv.set(key, { offset: writeOff, length: chunk.length, crc: crc32(chunk) });
+            writeOff += 4 + chunk.length;
+        }
+        for (const [key, value] of newChunks) {
+            chunks.push({ value });
+            liveKv.set(key, { offset: writeOff, length: value.length, crc: crc32(value) });
+            writeOff += 4 + value.length;
+        }
+        const hotEvents = latest.committed.slice(archiveCount);
+        const evtBytes = serializeEventSection(hotEvents);
+        const idxBytes = TEXT.encode(JSON.stringify(Object.fromEntries(liveKv)));
+        const evtOff = writeOff;
+        const kvIdxOff = evtOff + evtBytes.length;
+        const totalSize = kvIdxOff + idxBytes.length;
+        const out = new Uint8Array(totalSize);
+        const header = new Uint8Array(HEADER_SIZE);
+        writeHeaderV2(header, {
+            kvIndexOffset: kvIdxOff,
+            kvIndexLength: idxBytes.length,
+            kvChunkEnd: writeOff,
+            evtOffset: evtOff,
+            evtLength: evtBytes.length,
+            evtCount: hotEvents.length,
+            evtBaseId: latest.evtBaseId + archiveCount,
+        });
+        out.set(header, 0);
+        let cursor = HEADER_SIZE;
+        for (const chunk of chunks) {
+            out.set(u32be(chunk.value.length), cursor);
+            out.set(chunk.value, cursor + 4);
+            cursor += 4 + chunk.value.length;
+        }
+        out.set(evtBytes, evtOff);
+        out.set(idxBytes, kvIdxOff);
+        const tmpPath = `${this.storePath}.${process.pid}.${Date.now()}.archive.tmp`;
+        writeFileSync(tmpPath, out);
+        readHeaderV2(readFileSync(tmpPath).slice(0, HEADER_SIZE));
+        if (this.fh) {
+            await this.fh.close();
+            this.fh = null;
+        }
+        renameSync(tmpPath, this.storePath);
+        invalidateStoreSnapshotCache(this.storePath);
+        this.fh = await fsOpen(this.storePath, "r+");
+        this.kvIndex = liveKv;
+        this.kvChunkEnd = writeOff;
+        this.committed = hotEvents;
+        this.pending = [];
+        this.pendingKv.clear();
+        this.pendingDeletes.clear();
+        this.evtBaseId = latest.evtBaseId + archiveCount;
+        this.dirty = false;
+        return { batches_written: newBatches.length, events_archived: archiveCount, bytes_written: bytesWritten };
+    }
+    async getEntries(filter) {
+        const all = [...this.committed, ...this.pending];
+        const results = await this.getArchivedEntries(filter);
+        let id = this.evtBaseId + 1;
+        for (const rec of all) {
+            const currentId = id++;
+            const entry = eventRecordToEntry(rec, currentId);
+            if (!entry) {
+                this.skippedMalformedCount += 1;
+                console.error(JSON.stringify({ event: "store_entry_skipped_malformed", id: currentId, total: this.skippedMalformedCount }));
                 continue;
-            if (!filter?.includeDeleted && (entry.flags ?? 0) & 0x01)
+            }
+            if (!entryMatchesFilter(entry, filter))
                 continue;
             results.push(entry);
         }

package/dist/store/bgzf.d.ts

@@ -0,0 +1,4 @@
+export declare function bgzfEofBlock(): Uint8Array;
+export declare function bgzfCompress(uncompressed: Uint8Array): Uint8Array;
+export declare function bgzfDecompress(compressed: Uint8Array): Uint8Array;
+//# sourceMappingURL=bgzf.d.ts.map

package/dist/store/bgzf.js

@@ -0,0 +1,94 @@
+import { deflateRawSync, inflateRawSync } from "node:zlib";
+import { crc32 } from "./crc32.js";
+const MAX_BGZF_PAYLOAD = 65_535;
+const BGZF_EOF = new Uint8Array([
+    31, 139, 8, 4, 0, 0, 0, 0, 0, 255, 6, 0, 66, 67, 2, 0, 27, 0,
+    3, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+]);
+function u16le(buf, offset, value) {
+    new DataView(buf.buffer, buf.byteOffset).setUint16(offset, value, true);
+}
+function u32le(buf, offset, value) {
+    new DataView(buf.buffer, buf.byteOffset).setUint32(offset, value >>> 0, true);
+}
+function readU16le(buf, offset) {
+    return new DataView(buf.buffer, buf.byteOffset).getUint16(offset, true);
+}
+function readU32le(buf, offset) {
+    return new DataView(buf.buffer, buf.byteOffset).getUint32(offset, true);
+}
+function concat(parts) {
+    const total = parts.reduce((sum, part) => sum + part.length, 0);
+    const out = new Uint8Array(total);
+    let cursor = 0;
+    for (const part of parts) {
+        out.set(part, cursor);
+        cursor += part.length;
+    }
+    return out;
+}
+function makeBlock(payload) {
+    const compressed = deflateRawSync(payload);
+    const blockSize = 18 + compressed.length + 8;
+    if (blockSize > 65_536) {
+        throw new Error(`BGZF block too large after compression (${blockSize} bytes)`);
+    }
+    const out = new Uint8Array(blockSize);
+    out.set([31, 139, 8, 4, 0, 0, 0, 0, 0, 255, 6, 0, 66, 67, 2, 0], 0);
+    u16le(out, 16, blockSize - 1);
+    out.set(compressed, 18);
+    u32le(out, blockSize - 8, crc32(payload));
+    u32le(out, blockSize - 4, payload.length);
+    return out;
+}
+export function bgzfEofBlock() {
+    return BGZF_EOF.slice();
+}
+export function bgzfCompress(uncompressed) {
+    const blocks = [];
+    for (let offset = 0; offset < uncompressed.length || offset === 0; offset += MAX_BGZF_PAYLOAD) {
+        const chunk = uncompressed.slice(offset, Math.min(offset + MAX_BGZF_PAYLOAD, uncompressed.length));
+        blocks.push(makeBlock(chunk));
+        if (uncompressed.length === 0)
+            break;
+    }
+    blocks.push(bgzfEofBlock());
+    return concat(blocks);
+}
+export function bgzfDecompress(compressed) {
+    const chunks = [];
+    let offset = 0;
+    while (offset < compressed.length) {
+        if (compressed.length - offset < 18) {
+            throw new Error(`BGZF block truncated at offset ${offset}`);
+        }
+        if (compressed[offset] !== 31 ||
+            compressed[offset + 1] !== 139 ||
+            compressed[offset + 2] !== 8 ||
+            compressed[offset + 3] !== 4 ||
+            compressed[offset + 12] !== 66 ||
+            compressed[offset + 13] !== 67) {
+            throw new Error(`Invalid BGZF block header at offset ${offset}`);
+        }
+        const bsize = readU16le(compressed, offset + 16);
+        const blockLength = bsize + 1;
+        if (blockLength < 26 || offset + blockLength > compressed.length) {
+            throw new Error(`BGZF block extends past end of data at offset ${offset}`);
+        }
+        const compressedPayload = compressed.slice(offset + 18, offset + blockLength - 8);
+        const expectedCrc = readU32le(compressed, offset + blockLength - 8);
+        const expectedSize = readU32le(compressed, offset + blockLength - 4);
+        const inflated = inflateRawSync(compressedPayload);
+        if (inflated.length !== expectedSize) {
+            throw new Error(`BGZF ISIZE mismatch at offset ${offset} (expected=${expectedSize}, actual=${inflated.length})`);
+        }
+        if (crc32(inflated) !== expectedCrc) {
+            throw new Error(`BGZF CRC32 mismatch at offset ${offset}`);
+        }
+        if (inflated.length > 0)
+            chunks.push(inflated);
+        offset += blockLength;
+    }
+    return concat(chunks);
+}
+//# sourceMappingURL=bgzf.js.map

package/dist/store/bootstrap-store.d.ts

@@ -41,8 +41,11 @@ export interface BootstrapResult {
 }
 export declare function bootstrapStoreWorkspace(opts: BootstrapOptions): Promise<BootstrapResult>;
 export declare function repairWorkspace(workspaceRoot: string): Promise<string[]>;
-export declare function compactWorkspace(workspaceRoot: string): Promise<{
+export declare function compactWorkspace(workspaceRoot: string, options?: {
+    sweep?: boolean;
+}): Promise<{
     before: number;
     after: number;
+    sweep?: Awaited<ReturnType<typeof import("./cache-workspace.js")["cacheWorkspaceArtifacts"]>>;
 }>;
 //# sourceMappingURL=bootstrap-store.d.ts.map