@voybio/ace-swarm 0.1.0 → 0.2.0

package/dist/store/ace-packed-store.js

@@ -1,258 +1,437 @@
 /**
- * AcePackedStore
+ * AcePackedStore — ACEPACK v2
  *
- * Custom single-file store that implements Zarrita's AsyncReadable + AsyncWritable
- * interfaces over a packed binary file at `.agents/ACE/ace-state.zarr`.
+ * Single-file binary store. One file, no subdirectories, no companion files.
+ * Events grow in the columnar section indefinitely (Option A). compact() only
+ * reclaims dead space in the KV chunk region — it does NOT reset or archive
+ * events. Historical event replay across all past runs is always available.
  *
- * File format:
- *   [ Header  64 bytes ]   magic(8) version(4) flags(4) index_offset(8) index_length(8) reserved(32)
- *   [ Chunk region     ]   chunk_0: [uint32 length][bytes] ... chunk_N: [uint32 length][bytes]
- *   [ Index region     ]   UTF-8 JSON: { key → { offset, length } }
+ * File layout:
  *
- * Write protocol:
- *   1. Append new chunk to end of chunk region.
- *   2. Update in-memory index.
- *   3. On commit(): rewrite index region → update header → fsync.
+ *   ┌─────────────────────────────────────────────────────────┐
+ *   │ Header (128 bytes, big-endian)                           │
+ *   │   magic "ACEPACK\0" · version uint32 · flags uint32     │
+ *   │   kv_index_offset uint64 · kv_index_length uint64       │
+ *   │   kv_chunk_end uint64                                    │
+ *   │   evt_offset uint64 · evt_length uint64                  │
+ *   │   evt_count uint32 · evt_base_id uint32                  │
+ *   │   reserved (zeros)                                       │
+ *   ├─────────────────────────────────────────────────────────┤
+ *   │ KV Chunk Region (append-only, random-access)            │
+ *   │   knowledge/agents/{name}/{file} → instruction text     │
+ *   │   knowledge/skills/{name}/{file} → skill content        │
+ *   │   topology/{kind}                → JSON array           │
+ *   │   state/{...}                    → runtime state blobs  │
+ *   │   meta/{...}                     → schema version etc.  │
+ *   │   Each chunk: [uint32 length BE][bytes]                 │
+ *   ├─────────────────────────────────────────────────────────┤
+ *   │ Columnar Event Section (rewritten on every commit)      │
+ *   │   uint32    count                                        │
+ *   │   int64[N]  timestamps  (epoch ms, big-endian)          │
+ *   │   uint8[N]  kinds       (EntityKind enum)               │
+ *   │   uint8[N]  sources     (ContentSource enum)            │
+ *   │   uint8[N]  flags       (0x01=deleted)                  │
+ *   │   [pad to 4-byte alignment]                             │
+ *   │   uint32[N] pay_offsets (relative to pool start)        │
+ *   │   uint32[N] pay_lengths                                 │
+ *   │   uint32    pool_length                                  │
+ *   │   bytes[M]  payload pool (UTF-8 JSON: {key,payload,...})│
+ *   ├─────────────────────────────────────────────────────────┤
+ *   │ KV Index (JSON, rewritten on commit)                    │
+ *   │   { "knowledge/agents/ace-ops/AGENT.md": {offset, length}, ... }
+ *   └─────────────────────────────────────────────────────────┘
  *
- * Compaction:
- *   Write live chunks to temp file → validate → atomic rename.
+ * Event overhead: ~19 bytes fixed + payload (vs ~150 bytes full JSON).
+ * Timestamp/kind scanning reads only the fixed-width columns, never the pool.
  *
- * Locking:
- *   Advisory flock-style via a `.lock` sidecar file (PID + timestamp).
- *   Stale detection: lock held >30s and PID not alive → break lock.
+ * compact() removes dead KV space (overwritten keys) via atomic tmp-rename.
+ * Events are preserved across compact() — they accumulate for the workspace
+ * lifetime. See HOT_COLD_EVENT_TIERING.md for a future branch that adds
+ * BGZF batch archiving for very long-lived workspaces.
+ *
+ * Backward compat: v1 files (64-byte header) are migrated on first commit().
+ * v1 events live in core/log/ KV blobs and are pulled into the columnar section.
  */
-import { readFileSync, writeFileSync, existsSync, mkdirSync, renameSync, unlinkSync } from "fs";
-import { open as fsOpen } from "fs/promises";
-import { dirname } from "path";
-import { MAGIC, STORE_VERSION, HEADER_SIZE, ContentSource, EntityKind, } from "./types.js";
-const LOCK_TIMEOUT_MS = 30_000;
+import { readFileSync, writeFileSync, existsSync, mkdirSync, renameSync } from "node:fs";
+import { open as fsOpen } from "node:fs/promises";
+import { dirname } from "node:path";
+import { MAGIC, STORE_VERSION, HEADER_SIZE, HEADER_SIZE_V1, ContentSource, EntityKind, } from "./types.js";
+// ── Constants ─────────────────────────────────────────────────────────────────
+// Header field byte offsets (big-endian throughout)
+const H_KV_IDX_OFF = 16; // uint64: KV index offset
+const H_KV_IDX_LEN = 24; // uint64: KV index length
+const H_KV_CHUNK_END = 32; // uint64: end of KV chunk region (v2 only)
+const H_EVT_OFF = 40; // uint64: event section offset (v2 only)
+const H_EVT_LEN = 48; // uint64: event section length (v2 only)
+const H_EVT_CNT = 56; // uint32: event count (v2 only)
+const H_EVT_BASE = 60; // uint32: event base id (v2 only)
 const TEXT = new TextEncoder();
 const DECODE = new TextDecoder();
-function fnv1a(s) {
-    let h = 2166136261;
-    for (let i = 0; i < s.length; i++) {
-        h ^= s.charCodeAt(i);
-        h = Math.imul(h, 16777619) >>> 0;
-    }
-    return h >>> 0;
-}
-function uint32ToBytes(n) {
+// ── Byte helpers ──────────────────────────────────────────────────────────────
+function u32be(n) {
     const b = new Uint8Array(4);
     new DataView(b.buffer).setUint32(0, n, false);
     return b;
 }
-function bytesToUint32(b, offset = 0) {
-    return new DataView(b.buffer, b.byteOffset).getUint32(offset, false);
-}
-function uint64ToBytes(n) {
-    // JS numbers are safe up to 2^53 — sufficient for file offsets
+function u64be(n) {
     const b = new Uint8Array(8);
     const dv = new DataView(b.buffer);
     dv.setUint32(0, Math.floor(n / 2 ** 32), false);
     dv.setUint32(4, n >>> 0, false);
     return b;
 }
-function bytesToUint64(b, offset = 0) {
-    const dv = new DataView(b.buffer, b.byteOffset);
-    return dv.getUint32(offset, false) * 2 ** 32 + dv.getUint32(offset + 4, false);
-}
-async function readExact(handle, buffer, position) {
-    let offset = 0;
-    while (offset < buffer.length) {
-        const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, position + offset);
-        if (bytesRead <= 0) {
-            throw new Error("AcePackedStore: unexpected EOF while reading chunk");
-        }
-        offset += bytesRead;
-    }
-}
-async function writeExact(handle, buffer, position) {
-    const source = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
-    let offset = 0;
-    while (offset < source.length) {
-        const { bytesWritten } = await handle.write(source, offset, source.length - offset, position + offset);
-        if (bytesWritten <= 0) {
-            throw new Error("AcePackedStore: failed to write chunk");
-        }
-        offset += bytesWritten;
-    }
-}
-function acquireLock(lockPath) {
-    if (existsSync(lockPath)) {
-        const raw = readFileSync(lockPath, "utf8");
-        try {
-            const rec = JSON.parse(raw);
-            const age = Date.now() - rec.ts;
-            if (age < LOCK_TIMEOUT_MS) {
-                // Check if PID is alive
-                try {
-                    process.kill(rec.pid, 0);
-                    throw new Error(`AcePackedStore: locked by PID ${rec.pid} (${age}ms ago)`);
-                }
-                catch (e) {
-                    // PID not alive — stale lock, fall through to break it
-                    if (e.code !== "ESRCH")
-                        throw e;
-                }
-            }
-        }
-        catch {
-            // Unparseable lock — break it
-        }
-    }
-    writeFileSync(lockPath, JSON.stringify({ pid: process.pid, ts: Date.now() }), "utf8");
+function readU32(b, off = 0) {
+    return new DataView(b.buffer, b.byteOffset).getUint32(off, false);
 }
-function releaseLock(lockPath) {
-    if (existsSync(lockPath)) {
-        try {
-            unlinkSync(lockPath);
-        }
-        catch { /* ignore */ }
-    }
+function readU64(b, off = 0) {
+    const dv = new DataView(b.buffer, b.byteOffset);
+    return dv.getUint32(off, false) * 2 ** 32 + dv.getUint32(off + 4, false);
 }
-// ── Header serialisation ──────────────────────────────────────────────────────
-function writeHeader(buf, indexOffset, indexLength) {
-    // magic
+function writeHeaderV2(buf, h) {
     for (let i = 0; i < 8; i++)
         buf[i] = MAGIC.charCodeAt(i);
-    // version
-    buf.set(uint32ToBytes(STORE_VERSION), 8);
-    // flags
-    buf.set(uint32ToBytes(0), 12);
-    // index_offset
-    buf.set(uint64ToBytes(indexOffset), 16);
-    // index_length
-    buf.set(uint64ToBytes(indexLength), 24);
-    // reserved: zero-filled (already 0 on new Uint8Array)
+    buf.set(u32be(STORE_VERSION), 8);
+    buf.set(u32be(0), 12);
+    buf.set(u64be(h.kvIndexOffset), H_KV_IDX_OFF);
+    buf.set(u64be(h.kvIndexLength), H_KV_IDX_LEN);
+    buf.set(u64be(h.kvChunkEnd), H_KV_CHUNK_END);
+    buf.set(u64be(h.evtOffset), H_EVT_OFF);
+    buf.set(u64be(h.evtLength), H_EVT_LEN);
+    buf.set(u32be(h.evtCount), H_EVT_CNT);
+    buf.set(u32be(h.evtBaseId), H_EVT_BASE);
 }
-function readHeader(buf) {
+function readHeaderV2(buf) {
     const magic = DECODE.decode(buf.slice(0, 8));
     if (magic !== MAGIC)
-        throw new Error("AcePackedStore: invalid magic bytes — not an ACEPACK file");
+        throw new Error("AcePackedStore: invalid magic — not an ACEPACK file");
+    const version = readU32(buf, 8);
+    const isV2 = version >= 2;
     return {
-        version: bytesToUint32(buf, 8),
-        indexOffset: bytesToUint64(buf, 16),
-        indexLength: bytesToUint64(buf, 24),
+        version,
+        kvIndexOffset: readU64(buf, H_KV_IDX_OFF),
+        kvIndexLength: readU64(buf, H_KV_IDX_LEN),
+        kvChunkEnd: isV2 ? readU64(buf, H_KV_CHUNK_END) : readU64(buf, H_KV_IDX_OFF),
+        evtOffset: isV2 ? readU64(buf, H_EVT_OFF) : 0,
+        evtLength: isV2 ? readU64(buf, H_EVT_LEN) : 0,
+        evtCount: isV2 ? readU32(buf, H_EVT_CNT) : 0,
+        evtBaseId: isV2 ? readU32(buf, H_EVT_BASE) : 0,
     };
 }
+// ── Columnar event section serialiser ────────────────────────────────────────
+function serializeEventSection(events) {
+    const N = events.length;
+    const payloadBytes = events.map(e => TEXT.encode(e.blob));
+    const poolSize = payloadBytes.reduce((s, b) => s + b.length, 0);
+    const pad = (4 - (N % 4)) % 4;
+    // Layout: [4] count + [8N] ts + [N] kind + [N] source + [N] flags + [pad]
+    //       + [4N] pay_off + [4N] pay_len + [4] pool_len + [M] pool
+    const size = 4 + 8 * N + N + N + N + pad + 4 * N + 4 * N + 4 + poolSize;
+    const buf = new Uint8Array(size);
+    const dv = new DataView(buf.buffer);
+    let pos = 0;
+    dv.setUint32(pos, N, false);
+    pos += 4;
+    for (const e of events) {
+        dv.setUint32(pos, Math.floor(e.ts / 2 ** 32), false);
+        dv.setUint32(pos + 4, e.ts >>> 0, false);
+        pos += 8;
+    }
+    for (const e of events)
+        buf[pos++] = e.kind;
+    for (const e of events)
+        buf[pos++] = e.source;
+    for (const e of events)
+        buf[pos++] = e.flags;
+    pos += pad;
+    let poolOff = 0;
+    for (const pb of payloadBytes) {
+        dv.setUint32(pos, poolOff, false);
+        pos += 4;
+        poolOff += pb.length;
+    }
+    for (const pb of payloadBytes) {
+        dv.setUint32(pos, pb.length, false);
+        pos += 4;
+    }
+    dv.setUint32(pos, poolSize, false);
+    pos += 4;
+    for (const pb of payloadBytes) {
+        buf.set(pb, pos);
+        pos += pb.length;
+    }
+    return buf;
+}
+function deserializeEventSection(buf) {
+    if (buf.length < 4)
+        return [];
+    const dv = new DataView(buf.buffer, buf.byteOffset);
+    const N = dv.getUint32(0, false);
+    if (N === 0)
+        return [];
+    let pos = 4;
+    const ts = [];
+    for (let i = 0; i < N; i++) {
+        ts.push(dv.getUint32(pos, false) * 2 ** 32 + dv.getUint32(pos + 4, false));
+        pos += 8;
+    }
+    const kinds = [];
+    for (let i = 0; i < N; i++)
+        kinds.push(buf[pos++]);
+    const sources = [];
+    for (let i = 0; i < N; i++)
+        sources.push(buf[pos++]);
+    const flags = [];
+    for (let i = 0; i < N; i++)
+        flags.push(buf[pos++]);
+    pos += (4 - (N % 4)) % 4; // skip pad
+    const payOffs = [];
+    for (let i = 0; i < N; i++) {
+        payOffs.push(dv.getUint32(pos, false));
+        pos += 4;
+    }
+    const payLens = [];
+    for (let i = 0; i < N; i++) {
+        payLens.push(dv.getUint32(pos, false));
+        pos += 4;
+    }
+    pos += 4; // pool_length (implicit from payLens sum)
+    const poolStart = pos;
+    return Array.from({ length: N }, (_, i) => ({
+        ts: ts[i],
+        kind: kinds[i],
+        source: sources[i],
+        flags: flags[i],
+        blob: DECODE.decode(buf.slice(poolStart + payOffs[i], poolStart + payOffs[i] + payLens[i])),
+    }));
+}
+// ── File I/O helpers ──────────────────────────────────────────────────────────
+async function readExact(fh, buf, position) {
+    let off = 0;
+    while (off < buf.length) {
+        const { bytesRead } = await fh.read(buf, off, buf.length - off, position + off);
+        if (bytesRead <= 0)
+            throw new Error("AcePackedStore: unexpected EOF");
+        off += bytesRead;
+    }
+}
+async function writeExact(fh, data, position) {
+    let off = 0;
+    while (off < data.length) {
+        const { bytesWritten } = await fh.write(data, off, data.length - off, position + off);
+        if (bytesWritten <= 0)
+            throw new Error("AcePackedStore: write failed");
+        off += bytesWritten;
+    }
+}
 // ── AcePackedStore ────────────────────────────────────────────────────────────
 export class AcePackedStore {
     storePath = "";
-    lockPath = "";
     readOnly = false;
-    index = new Map();
-    chunkEnd = HEADER_SIZE; // byte offset immediately after last chunk
     fh = null;
-    dirty = false;
-    entryCounter = 0;
-    // ── Lifecycle ─────────────────────────────────────────────────────────────
+    // KV region
+    kvIndex = new Map();
+    kvChunkEnd = HEADER_SIZE; // end of KV chunk region in file
+    // Event log (in-memory)
+    committed = []; // loaded from file
+    pending = []; // written since last commit
+    evtBaseId = 0; // events [0..evtBaseId] are in the archive
+    // ── Lifecycle ───────────────────────────────────────────────────────────────
     async open(path, opts = {}) {
         this.storePath = path;
-        this.lockPath = path + ".lock";
         this.readOnly = opts.readOnly ?? false;
-        if (!this.readOnly) {
-            acquireLock(this.lockPath);
-        }
         if (!existsSync(path)) {
             if (this.readOnly)
-                throw new Error(`AcePackedStore: store not found at ${path}`);
-            await this._initNewStore(path);
+                throw new Error(`AcePackedStore: file not found: ${path}`);
+            await this._initNew(path);
         }
         else {
-            await this._loadExistingStore(path);
+            await this._loadExisting(path);
         }
         this.fh = await fsOpen(path, this.readOnly ? "r" : "r+");
     }
-    async _initNewStore(path) {
+    async _initNew(path) {
         mkdirSync(dirname(path), { recursive: true });
+        const evtBytes = serializeEventSection([]);
+        const idxBytes = TEXT.encode("{}");
+        const evtOff = HEADER_SIZE;
+        const kvIdxOff = evtOff + evtBytes.length;
+        const totalSize = kvIdxOff + idxBytes.length;
         const header = new Uint8Array(HEADER_SIZE);
-        // Index starts immediately after header (empty at init)
-        const emptyIndex = TEXT.encode("{}");
-        writeHeader(header, HEADER_SIZE, emptyIndex.length);
-        const buf = new Uint8Array(HEADER_SIZE + emptyIndex.length);
-        buf.set(header);
-        buf.set(emptyIndex, HEADER_SIZE);
-        writeFileSync(path, buf);
-        this.chunkEnd = HEADER_SIZE;
-        this.index = new Map();
-    }
-    async _loadExistingStore(path) {
+        writeHeaderV2(header, {
+            kvIndexOffset: kvIdxOff,
+            kvIndexLength: idxBytes.length,
+            kvChunkEnd: HEADER_SIZE,
+            evtOffset: evtOff,
+            evtLength: evtBytes.length,
+            evtCount: 0,
+            evtBaseId: 0,
+        });
+        const out = new Uint8Array(totalSize);
+        out.set(header, 0);
+        out.set(evtBytes, evtOff);
+        out.set(idxBytes, kvIdxOff);
+        writeFileSync(path, out);
+        this.kvChunkEnd = HEADER_SIZE;
+        this.kvIndex = new Map();
+        this.committed = [];
+        this.pending = [];
+        this.evtBaseId = 0;
+    }
+    async _loadExisting(path) {
         const file = readFileSync(path);
-        if (file.length < HEADER_SIZE)
+        if (file.length < HEADER_SIZE_V1)
             throw new Error("AcePackedStore: file too short");
-        const { indexOffset, indexLength } = readHeader(file.slice(0, HEADER_SIZE));
-        const idxBytes = file.slice(indexOffset, indexOffset + indexLength);
-        const idxJson = JSON.parse(DECODE.decode(idxBytes));
-        this.index = new Map(Object.entries(idxJson));
-        // chunkEnd = start of index region (chunks live between header and index)
-        this.chunkEnd = indexOffset;
-        // Re-derive entry counter from unified log entries
-        const logKeys = [...this.index.keys()].filter(k => k.startsWith("core/log/"));
-        this.entryCounter = logKeys.length;
+        const hdr = readHeaderV2(file.slice(0, Math.max(HEADER_SIZE, file.length)));
+        const isV1 = hdr.version < 2;
+        // Load KV index
+        const idxRaw = file.slice(hdr.kvIndexOffset, hdr.kvIndexOffset + hdr.kvIndexLength);
+        const idxObj = JSON.parse(DECODE.decode(idxRaw));
+        this.kvIndex = new Map(Object.entries(idxObj));
+        this.kvChunkEnd = hdr.kvChunkEnd;
+        if (isV1) {
+            // v1 → v2 migration: pull events from core/log/ KV entries
+            const logKeys = [...this.kvIndex.keys()]
+                .filter(k => k.startsWith("core/log/") && k !== "core/log/__seq")
+                .sort();
+            this.committed = [];
+            for (const lk of logKeys) {
+                const entry = await this._readKvBlobDirect(file, lk);
+                if (!entry)
+                    continue;
+                try {
+                    const e = JSON.parse(entry);
+                    this.committed.push({
+                        ts: e.ts,
+                        kind: e.kind,
+                        source: e.content_source,
+                        flags: e.flags ?? 0,
+                        blob: JSON.stringify({ key: e.key, payload: e.payload, parent_id: e.parent_id }),
+                    });
+                }
+                catch { /* skip malformed */ }
+            }
+            // Remove core/log/ and meta/log_entry_seq from KV (they go to columnar section)
+            for (const k of [...this.kvIndex.keys()]) {
+                if (k.startsWith("core/log/") || k === "meta/log_entry_seq")
+                    this.kvIndex.delete(k);
+            }
+            this.evtBaseId = 0;
+            // force commit to write v2 format (pending is non-empty from migration)
+        }
+        else {
+            // v2: load event section
+            if (hdr.evtLength > 0) {
+                const evtBuf = file.slice(hdr.evtOffset, hdr.evtOffset + hdr.evtLength);
+                this.committed = deserializeEventSection(evtBuf);
+            }
+            else {
+                this.committed = [];
+            }
+            this.evtBaseId = hdr.evtBaseId;
+        }
+        this.pending = [];
+    }
+    /** Read a KV blob directly from a loaded file buffer (used during migration). */
+    async _readKvBlobDirect(file, key) {
+        const entry = this.kvIndex.get(key);
+        if (!entry)
+            return undefined;
+        const start = entry.offset + 4;
+        const end = start + entry.length;
+        if (end > file.length)
+            return undefined;
+        return DECODE.decode(file.slice(start, end));
     }
     async commit() {
-        if (this.readOnly || !this.fh || !this.dirty)
+        if (this.readOnly || !this.fh)
             return;
-        const idxJson = JSON.stringify(Object.fromEntries(this.index));
+        // Merge pending into committed
+        this.committed.push(...this.pending);
+        this.pending = [];
+        // Serialize event section
+        const evtBytes = serializeEventSection(this.committed);
+        // Write KV index after event section
+        const idxJson = JSON.stringify(Object.fromEntries(this.kvIndex));
         const idxBytes = TEXT.encode(idxJson);
-        // Write index at current chunkEnd
-        await writeExact(this.fh, idxBytes, this.chunkEnd);
+        const evtOff = this.kvChunkEnd;
+        const kvIdxOff = evtOff + evtBytes.length;
+        await writeExact(this.fh, evtBytes, evtOff);
+        await writeExact(this.fh, idxBytes, kvIdxOff);
+        await this.fh.truncate(kvIdxOff + idxBytes.length);
         // Update header
         const header = new Uint8Array(HEADER_SIZE);
-        writeHeader(header, this.chunkEnd, idxBytes.length);
+        writeHeaderV2(header, {
+            kvIndexOffset: kvIdxOff,
+            kvIndexLength: idxBytes.length,
+            kvChunkEnd: this.kvChunkEnd,
+            evtOffset: evtOff,
+            evtLength: evtBytes.length,
+            evtCount: this.committed.length,
+            evtBaseId: this.evtBaseId,
+        });
         await writeExact(this.fh, header, 0);
-        // Truncate any stale trailing bytes
-        await this.fh.truncate(this.chunkEnd + idxBytes.length);
         await this.fh.datasync();
-        this.dirty = false;
     }
+    /**
+     * Compacts the KV chunk region — removes dead space left by overwritten keys.
+     * Events are NOT reset or archived; they accumulate in the columnar section
+     * for the workspace lifetime (Option A — grow forever).
+     * See HOT_COLD_EVENT_TIERING.md for the future branch that adds BGZF archiving.
+     */
     async compact() {
-        if (!this.storePath)
+        if (!this.storePath || this.readOnly)
             return;
-        const lockPath = this.lockPath;
-        const tmpPath = this.storePath + ".tmp";
-        // Write live chunks to temp file
-        const liveIndex = new Map();
-        let writeOffset = HEADER_SIZE;
+        // Flush pending events into committed so they survive the rewrite
+        this.committed.push(...this.pending);
+        this.pending = [];
+        // Rebuild KV chunk region with only live keys (removes dead space)
         const srcFile = readFileSync(this.storePath);
+        const liveKv = new Map();
         const chunks = [];
-        for (const [key, { offset, length }] of this.index) {
-            // offset points to the uint32 length prefix; skip 4 bytes to get pure data
-            const chunk = srcFile.slice(offset + 4, offset + 4 + length);
-            chunks.push(chunk);
-            liveIndex.set(key, { offset: writeOffset, length });
-            writeOffset += 4 + length; // uint32 length prefix + data
+        let writeOff = HEADER_SIZE;
+        for (const [key, { offset, length }] of this.kvIndex) {
+            chunks.push(srcFile.slice(offset + 4, offset + 4 + length));
+            liveKv.set(key, { offset: writeOff, length });
+            writeOff += 4 + length;
         }
-        const idxJson = JSON.stringify(Object.fromEntries(liveIndex));
-        const idxBytes = TEXT.encode(idxJson);
-        const totalSize = HEADER_SIZE + chunks.reduce((s, c) => s + 4 + c.length, 0) + idxBytes.length;
+        this.kvIndex = liveKv;
+        this.kvChunkEnd = writeOff;
+        // Serialize events (preserved in full) + new KV index
+        const evtBytes = serializeEventSection(this.committed);
+        const idxBytes = TEXT.encode(JSON.stringify(Object.fromEntries(liveKv)));
+        const evtOff = writeOff;
+        const kvIdxOff = evtOff + evtBytes.length;
+        const totalSize = kvIdxOff + idxBytes.length;
         const out = new Uint8Array(totalSize);
         const header = new Uint8Array(HEADER_SIZE);
-        writeHeader(header, writeOffset, idxBytes.length);
-        out.set(header);
+        writeHeaderV2(header, {
+            kvIndexOffset: kvIdxOff,
+            kvIndexLength: idxBytes.length,
+            kvChunkEnd: writeOff,
+            evtOffset: evtOff,
+            evtLength: evtBytes.length,
+            evtCount: this.committed.length,
+            evtBaseId: this.evtBaseId,
+        });
+        out.set(header, 0);
         let pos = HEADER_SIZE;
         for (const chunk of chunks) {
-            out.set(uint32ToBytes(chunk.length), pos);
-            out.set(chunk, pos + 4);
-            pos += 4 + chunk.length;
+            out.set(u32be(chunk.length), pos);
+            pos += 4;
+            out.set(chunk, pos);
+            pos += chunk.length;
         }
-        out.set(idxBytes, pos);
+        out.set(evtBytes, evtOff);
+        out.set(idxBytes, kvIdxOff);
+        const tmpPath = this.storePath + ".tmp";
         writeFileSync(tmpPath, out);
-        // Validate temp file magic
-        const tmpBuf = readFileSync(tmpPath);
-        readHeader(tmpBuf.slice(0, HEADER_SIZE)); // throws if invalid
-        // Atomic rename
+        readHeaderV2(readFileSync(tmpPath).slice(0, HEADER_SIZE)); // validate
         if (this.fh) {
             await this.fh.close();
             this.fh = null;
         }
         renameSync(tmpPath, this.storePath);
-        releaseLock(lockPath);
-        acquireLock(lockPath);
         this.fh = await fsOpen(this.storePath, "r+");
-        this.index = liveIndex;
-        this.chunkEnd = writeOffset;
-        this.dirty = false;
     }
     async close() {
         await this.commit();
@@ -260,83 +439,95 @@ export class AcePackedStore {
             await this.fh.close();
             this.fh = null;
         }
-        if (!this.readOnly)
-            releaseLock(this.lockPath);
     }
-    // ── Zarrita protocol ──────────────────────────────────────────────────────
+    // ── Core KV ─────────────────────────────────────────────────────────────────
     async get(key) {
-        const entry = this.index.get(key);
-        if (!entry)
+        const entry = this.kvIndex.get(key);
+        if (!entry || !this.fh)
             return undefined;
-        if (!this.fh)
-            throw new Error("AcePackedStore: store not open");
-        const dataBuf = Buffer.alloc(entry.length);
-        await readExact(this.fh, dataBuf, entry.offset + 4);
-        return dataBuf;
+        const buf = new Uint8Array(entry.length);
+        await readExact(this.fh, buf, entry.offset + 4);
+        return buf;
     }
     async set(key, value) {
         if (this.readOnly)
-            throw new Error("AcePackedStore: cannot write — store opened read-only");
+            throw new Error("AcePackedStore: read-only");
         if (!this.fh)
-            throw new Error("AcePackedStore: store not open");
-        const lenPrefix = uint32ToBytes(value.length);
-        const chunkOffset = this.chunkEnd;
-        await writeExact(this.fh, lenPrefix, chunkOffset);
-        await writeExact(this.fh, value, chunkOffset + 4);
-        this.index.set(key, { offset: chunkOffset, length: value.length });
-        this.chunkEnd += 4 + value.length;
-        this.dirty = true;
+            throw new Error("AcePackedStore: not open");
+        // Append chunk to KV chunk region (at kvChunkEnd)
+        await writeExact(this.fh, u32be(value.length), this.kvChunkEnd);
+        await writeExact(this.fh, value, this.kvChunkEnd + 4);
+        this.kvIndex.set(key, { offset: this.kvChunkEnd, length: value.length });
+        this.kvChunkEnd += 4 + value.length;
     }
     async delete(key) {
-        const existed = this.index.has(key);
-        this.index.delete(key);
-        if (existed)
-            this.dirty = true;
+        const existed = this.kvIndex.has(key);
+        this.kvIndex.delete(key);
         return existed;
     }
     async *list() {
-        for (const key of this.index.keys())
+        for (const key of this.kvIndex.keys())
             yield key;
     }
-    // ── JSON / Blob convenience ───────────────────────────────────────────────
+    // ── JSON / Blob convenience ──────────────────────────────────────────────────
     async getJSON(key) {
         const raw = await this.get(key);
         if (!raw)
             return undefined;
-        return JSON.parse(DECODE.decode(raw));
+        try {
+            return JSON.parse(DECODE.decode(raw));
+        }
+        catch {
+            return undefined;
+        }
     }
     async setJSON(key, value) {
         await this.set(key, TEXT.encode(JSON.stringify(value)));
     }
     async getBlob(key) {
         const raw = await this.get(key);
-        if (!raw)
-            return undefined;
-        return DECODE.decode(raw);
+        return raw ? DECODE.decode(raw) : undefined;
     }
     async setBlob(key, content) {
         await this.set(key, TEXT.encode(content));
     }
-    // ── Unified entry log ─────────────────────────────────────────────────────
+    // ── Unified event log ────────────────────────────────────────────────────────
     async appendEntry(entry) {
-        const seqKey = "meta/log_entry_seq";
-        const current = (await this.getJSON(seqKey)) ?? this.entryCounter;
-        const id = current + 1;
-        this.entryCounter = id;
-        await this.setJSON(seqKey, id);
-        const full = { ...entry, id, ts: Date.now() };
-        const logKey = `core/log/${String(id).padStart(12, "0")}`;
-        await this.setJSON(logKey, full);
-        return full;
+        const id = this.evtBaseId + this.committed.length + this.pending.length + 1;
+        const ts = Date.now();
+        const rec = {
+            ts,
+            kind: entry.kind,
+            source: entry.content_source,
+            flags: entry.flags ?? 0,
+            blob: JSON.stringify({ key: entry.key, payload: entry.payload, parent_id: entry.parent_id }),
+        };
+        this.pending.push(rec);
+        return { id, ts, kind: entry.kind, content_source: entry.content_source, key: entry.key, payload: entry.payload, flags: entry.flags };
     }
     async getEntries(filter) {
+        const all = [...this.committed, ...this.pending];
         const results = [];
-        for (const key of this.index.keys()) {
-            if (!key.startsWith("core/log/") || key === "core/log/__seq")
-                continue;
-            const entry = await this.getJSON(key);
-            if (!entry)
+        let id = this.evtBaseId + 1;
+        for (const rec of all) {
+            let parsed;
+            try {
+                parsed = JSON.parse(rec.blob);
+            }
+            catch {
+                id++;
                 continue;
+            }
+            const entry = {
+                id: id++,
+                ts: rec.ts,
+                kind: rec.kind,
+                content_source: rec.source,
+                key: parsed.key,
+                payload: parsed.payload,
+                parent_id: parsed.parent_id,
+                flags: rec.flags,
+            };
             if (filter?.kind !== undefined && entry.kind !== filter.kind)
                 continue;
             if (filter?.content_source !== undefined && entry.content_source !== filter.content_source)
@@ -349,80 +540,76 @@ export class AcePackedStore {
                 continue;
             results.push(entry);
         }
-        return results.sort((a, b) => a.id - b.id);
+        return results;
     }
-    async getEntry(key) {
-        return this.getJSON(key);
+    async getEntry(_key) {
+        // Events are no longer in KV — scan by id if key is a core/log/ path
+        const match = _key.match(/^core\/log\/(\d+)$/);
+        if (!match)
+            return undefined;
+        const targetId = parseInt(match[1], 10);
+        const all = await this.getEntries();
+        return all.find(e => e.id === targetId);
     }
-    // ── Topology ──────────────────────────────────────────────────────────────
+    // ── Topology ─────────────────────────────────────────────────────────────────
     async getTopology(kind) {
         return (await this.getJSON(`topology/${kind}`)) ?? [];
     }
     async setTopology(kind, entries) {
         await this.setJSON(`topology/${kind}`, entries);
     }
-    // ── Knowledge: agents ─────────────────────────────────────────────────────
+    // ── Knowledge: agents ────────────────────────────────────────────────────────
     async getAgentInstruction(agent, file) {
         return this.getBlob(`knowledge/agents/${agent}/${file}`);
     }
     async setAgentInstruction(agent, file, content, source = ContentSource.Package) {
         const key = `knowledge/agents/${agent}/${file}`;
         await this.setBlob(key, content);
-        await this.appendEntry({
-            kind: EntityKind.AgentInstruction,
-            content_source: source,
-            key,
-            payload: { agent, file, length: content.length },
-        });
+        await this.appendEntry({ kind: EntityKind.AgentInstruction, content_source: source, key, payload: { agent, file, length: content.length } });
     }
     async listAgents() {
         const agents = new Set();
-        for (const key of this.index.keys()) {
-            if (key.startsWith("knowledge/agents/")) {
-                const parts = key.split("/");
-                if (parts[2])
-                    agents.add(parts[2]);
+        for (const k of this.kvIndex.keys()) {
+            if (k.startsWith("knowledge/agents/")) {
+                const p = k.split("/");
+                if (p[2])
+                    agents.add(p[2]);
             }
         }
         return [...agents].sort();
     }
-    // ── Knowledge: skills ─────────────────────────────────────────────────────
+    // ── Knowledge: skills ────────────────────────────────────────────────────────
     async getSkillContent(skill, file) {
         return this.getBlob(`knowledge/skills/${skill}/${file}`);
     }
     async setSkillContent(skill, file, content, source = ContentSource.Package) {
         const key = `knowledge/skills/${skill}/${file}`;
         await this.setBlob(key, content);
-        await this.appendEntry({
-            kind: EntityKind.SkillDefinition,
-            content_source: source,
-            key,
-            payload: { skill, file, length: content.length },
-        });
+        await this.appendEntry({ kind: EntityKind.SkillDefinition, content_source: source, key, payload: { skill, file, length: content.length } });
     }
     async listSkills() {
         const skills = new Set();
-        for (const key of this.index.keys()) {
-            if (key.startsWith("knowledge/skills/")) {
-                const parts = key.split("/");
-                if (parts[2])
-                    skills.add(parts[2]);
+        for (const k of this.kvIndex.keys()) {
+            if (k.startsWith("knowledge/skills/")) {
+                const p = k.split("/");
+                if (p[2])
+                    skills.add(p[2]);
             }
         }
         return [...skills].sort();
     }
-    // ── Internal helpers ──────────────────────────────────────────────────────
-    /** Dead space ratio — used to decide if compaction is needed */
+    // ── Introspection ─────────────────────────────────────────────────────────────
+    /** Dead space ratio in the KV chunk region. Used to decide if compaction is needed. */
     get deadSpaceRatio() {
-        if (!existsSync(this.storePath))
+        const totalKv = this.kvChunkEnd - HEADER_SIZE;
+        if (totalKv <= 0)
             return 0;
-        const totalSize = this.chunkEnd;
-        const liveSize = HEADER_SIZE + [...this.index.values()].reduce((s, e) => s + 4 + e.length, 0);
-        return totalSize > 0 ? 1 - liveSize / totalSize : 0;
+        const liveKv = [...this.kvIndex.values()].reduce((s, e) => s + 4 + e.length, 0);
+        return Math.max(0, 1 - liveKv / totalKv);
     }
-    /** Number of entries in the unified log */
+    /** Total events in log (committed + pending). */
     get entryCount() {
-        return this.entryCounter;
+        return this.committed.length + this.pending.length;
     }
 }
 // ── Factory ────────────────────────────────────────────────────────────────────