@voybio/ace-swarm 0.1.0

package/dist/store/ace-packed-store.js

@@ -0,0 +1,434 @@
+/**
+ * AcePackedStore
+ *
+ * Custom single-file store that implements Zarrita's AsyncReadable + AsyncWritable
+ * interfaces over a packed binary file at `.agents/ACE/ace-state.zarr`.
+ *
+ * File format:
+ *   [ Header  64 bytes ]   magic(8) version(4) flags(4) index_offset(8) index_length(8) reserved(32)
+ *   [ Chunk region     ]   chunk_0: [uint32 length][bytes] ... chunk_N: [uint32 length][bytes]
+ *   [ Index region     ]   UTF-8 JSON: { key → { offset, length } }
+ *
+ * Write protocol:
+ *   1. Append new chunk to end of chunk region.
+ *   2. Update in-memory index.
+ *   3. On commit(): rewrite index region → update header → fsync.
+ *
+ * Compaction:
+ *   Write live chunks to temp file → validate → atomic rename.
+ *
+ * Locking:
+ *   Advisory flock-style via a `.lock` sidecar file (PID + timestamp).
+ *   Stale detection: lock held >30s and PID not alive → break lock.
+ */
+import { readFileSync, writeFileSync, existsSync, mkdirSync, renameSync, unlinkSync } from "fs";
+import { open as fsOpen } from "fs/promises";
+import { dirname } from "path";
+import { MAGIC, STORE_VERSION, HEADER_SIZE, ContentSource, EntityKind, } from "./types.js";
+const LOCK_TIMEOUT_MS = 30_000;
+const TEXT = new TextEncoder();
+const DECODE = new TextDecoder();
+function fnv1a(s) {
+    let h = 2166136261;
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 16777619) >>> 0;
+    }
+    return h >>> 0;
+}
+function uint32ToBytes(n) {
+    const b = new Uint8Array(4);
+    new DataView(b.buffer).setUint32(0, n, false);
+    return b;
+}
+function bytesToUint32(b, offset = 0) {
+    return new DataView(b.buffer, b.byteOffset).getUint32(offset, false);
+}
+function uint64ToBytes(n) {
+    // JS numbers are safe up to 2^53 — sufficient for file offsets
+    const b = new Uint8Array(8);
+    const dv = new DataView(b.buffer);
+    dv.setUint32(0, Math.floor(n / 2 ** 32), false);
+    dv.setUint32(4, n >>> 0, false);
+    return b;
+}
+function bytesToUint64(b, offset = 0) {
+    const dv = new DataView(b.buffer, b.byteOffset);
+    return dv.getUint32(offset, false) * 2 ** 32 + dv.getUint32(offset + 4, false);
+}
+async function readExact(handle, buffer, position) {
+    let offset = 0;
+    while (offset < buffer.length) {
+        const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, position + offset);
+        if (bytesRead <= 0) {
+            throw new Error("AcePackedStore: unexpected EOF while reading chunk");
+        }
+        offset += bytesRead;
+    }
+}
+async function writeExact(handle, buffer, position) {
+    const source = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
+    let offset = 0;
+    while (offset < source.length) {
+        const { bytesWritten } = await handle.write(source, offset, source.length - offset, position + offset);
+        if (bytesWritten <= 0) {
+            throw new Error("AcePackedStore: failed to write chunk");
+        }
+        offset += bytesWritten;
+    }
+}
+function acquireLock(lockPath) {
+    if (existsSync(lockPath)) {
+        const raw = readFileSync(lockPath, "utf8");
+        try {
+            const rec = JSON.parse(raw);
+            const age = Date.now() - rec.ts;
+            if (age < LOCK_TIMEOUT_MS) {
+                // Check if PID is alive
+                try {
+                    process.kill(rec.pid, 0);
+                    throw new Error(`AcePackedStore: locked by PID ${rec.pid} (${age}ms ago)`);
+                }
+                catch (e) {
+                    // PID not alive — stale lock, fall through to break it
+                    if (e.code !== "ESRCH")
+                        throw e;
+                }
+            }
+        }
+        catch {
+            // Unparseable lock — break it
+        }
+    }
+    writeFileSync(lockPath, JSON.stringify({ pid: process.pid, ts: Date.now() }), "utf8");
+}
+function releaseLock(lockPath) {
+    if (existsSync(lockPath)) {
+        try {
+            unlinkSync(lockPath);
+        }
+        catch { /* ignore */ }
+    }
+}
+// ── Header serialisation ──────────────────────────────────────────────────────
+function writeHeader(buf, indexOffset, indexLength) {
+    // magic
+    for (let i = 0; i < 8; i++)
+        buf[i] = MAGIC.charCodeAt(i);
+    // version
+    buf.set(uint32ToBytes(STORE_VERSION), 8);
+    // flags
+    buf.set(uint32ToBytes(0), 12);
+    // index_offset
+    buf.set(uint64ToBytes(indexOffset), 16);
+    // index_length
+    buf.set(uint64ToBytes(indexLength), 24);
+    // reserved: zero-filled (already 0 on new Uint8Array)
+}
+function readHeader(buf) {
+    const magic = DECODE.decode(buf.slice(0, 8));
+    if (magic !== MAGIC)
+        throw new Error("AcePackedStore: invalid magic bytes — not an ACEPACK file");
+    return {
+        version: bytesToUint32(buf, 8),
+        indexOffset: bytesToUint64(buf, 16),
+        indexLength: bytesToUint64(buf, 24),
+    };
+}
+// ── AcePackedStore ────────────────────────────────────────────────────────────
+export class AcePackedStore {
+    storePath = "";
+    lockPath = "";
+    readOnly = false;
+    index = new Map();
+    chunkEnd = HEADER_SIZE; // byte offset immediately after last chunk
+    fh = null;
+    dirty = false;
+    entryCounter = 0;
+    // ── Lifecycle ─────────────────────────────────────────────────────────────
+    async open(path, opts = {}) {
+        this.storePath = path;
+        this.lockPath = path + ".lock";
+        this.readOnly = opts.readOnly ?? false;
+        if (!this.readOnly) {
+            acquireLock(this.lockPath);
+        }
+        if (!existsSync(path)) {
+            if (this.readOnly)
+                throw new Error(`AcePackedStore: store not found at ${path}`);
+            await this._initNewStore(path);
+        }
+        else {
+            await this._loadExistingStore(path);
+        }
+        this.fh = await fsOpen(path, this.readOnly ? "r" : "r+");
+    }
+    async _initNewStore(path) {
+        mkdirSync(dirname(path), { recursive: true });
+        const header = new Uint8Array(HEADER_SIZE);
+        // Index starts immediately after header (empty at init)
+        const emptyIndex = TEXT.encode("{}");
+        writeHeader(header, HEADER_SIZE, emptyIndex.length);
+        const buf = new Uint8Array(HEADER_SIZE + emptyIndex.length);
+        buf.set(header);
+        buf.set(emptyIndex, HEADER_SIZE);
+        writeFileSync(path, buf);
+        this.chunkEnd = HEADER_SIZE;
+        this.index = new Map();
+    }
+    async _loadExistingStore(path) {
+        const file = readFileSync(path);
+        if (file.length < HEADER_SIZE)
+            throw new Error("AcePackedStore: file too short");
+        const { indexOffset, indexLength } = readHeader(file.slice(0, HEADER_SIZE));
+        const idxBytes = file.slice(indexOffset, indexOffset + indexLength);
+        const idxJson = JSON.parse(DECODE.decode(idxBytes));
+        this.index = new Map(Object.entries(idxJson));
+        // chunkEnd = start of index region (chunks live between header and index)
+        this.chunkEnd = indexOffset;
+        // Re-derive entry counter from unified log entries
+        const logKeys = [...this.index.keys()].filter(k => k.startsWith("core/log/"));
+        this.entryCounter = logKeys.length;
+    }
+    async commit() {
+        if (this.readOnly || !this.fh || !this.dirty)
+            return;
+        const idxJson = JSON.stringify(Object.fromEntries(this.index));
+        const idxBytes = TEXT.encode(idxJson);
+        // Write index at current chunkEnd
+        await writeExact(this.fh, idxBytes, this.chunkEnd);
+        // Update header
+        const header = new Uint8Array(HEADER_SIZE);
+        writeHeader(header, this.chunkEnd, idxBytes.length);
+        await writeExact(this.fh, header, 0);
+        // Truncate any stale trailing bytes
+        await this.fh.truncate(this.chunkEnd + idxBytes.length);
+        await this.fh.datasync();
+        this.dirty = false;
+    }
+    async compact() {
+        if (!this.storePath)
+            return;
+        const lockPath = this.lockPath;
+        const tmpPath = this.storePath + ".tmp";
+        // Write live chunks to temp file
+        const liveIndex = new Map();
+        let writeOffset = HEADER_SIZE;
+        const srcFile = readFileSync(this.storePath);
+        const chunks = [];
+        for (const [key, { offset, length }] of this.index) {
+            // offset points to the uint32 length prefix; skip 4 bytes to get pure data
+            const chunk = srcFile.slice(offset + 4, offset + 4 + length);
+            chunks.push(chunk);
+            liveIndex.set(key, { offset: writeOffset, length });
+            writeOffset += 4 + length; // uint32 length prefix + data
+        }
+        const idxJson = JSON.stringify(Object.fromEntries(liveIndex));
+        const idxBytes = TEXT.encode(idxJson);
+        const totalSize = HEADER_SIZE + chunks.reduce((s, c) => s + 4 + c.length, 0) + idxBytes.length;
+        const out = new Uint8Array(totalSize);
+        const header = new Uint8Array(HEADER_SIZE);
+        writeHeader(header, writeOffset, idxBytes.length);
+        out.set(header);
+        let pos = HEADER_SIZE;
+        for (const chunk of chunks) {
+            out.set(uint32ToBytes(chunk.length), pos);
+            out.set(chunk, pos + 4);
+            pos += 4 + chunk.length;
+        }
+        out.set(idxBytes, pos);
+        writeFileSync(tmpPath, out);
+        // Validate temp file magic
+        const tmpBuf = readFileSync(tmpPath);
+        readHeader(tmpBuf.slice(0, HEADER_SIZE)); // throws if invalid
+        // Atomic rename
+        if (this.fh) {
+            await this.fh.close();
+            this.fh = null;
+        }
+        renameSync(tmpPath, this.storePath);
+        releaseLock(lockPath);
+        acquireLock(lockPath);
+        this.fh = await fsOpen(this.storePath, "r+");
+        this.index = liveIndex;
+        this.chunkEnd = writeOffset;
+        this.dirty = false;
+    }
+    async close() {
+        await this.commit();
+        if (this.fh) {
+            await this.fh.close();
+            this.fh = null;
+        }
+        if (!this.readOnly)
+            releaseLock(this.lockPath);
+    }
+    // ── Zarrita protocol ──────────────────────────────────────────────────────
+    async get(key) {
+        const entry = this.index.get(key);
+        if (!entry)
+            return undefined;
+        if (!this.fh)
+            throw new Error("AcePackedStore: store not open");
+        const dataBuf = Buffer.alloc(entry.length);
+        await readExact(this.fh, dataBuf, entry.offset + 4);
+        return dataBuf;
+    }
+    async set(key, value) {
+        if (this.readOnly)
+            throw new Error("AcePackedStore: cannot write — store opened read-only");
+        if (!this.fh)
+            throw new Error("AcePackedStore: store not open");
+        const lenPrefix = uint32ToBytes(value.length);
+        const chunkOffset = this.chunkEnd;
+        await writeExact(this.fh, lenPrefix, chunkOffset);
+        await writeExact(this.fh, value, chunkOffset + 4);
+        this.index.set(key, { offset: chunkOffset, length: value.length });
+        this.chunkEnd += 4 + value.length;
+        this.dirty = true;
+    }
+    async delete(key) {
+        const existed = this.index.has(key);
+        this.index.delete(key);
+        if (existed)
+            this.dirty = true;
+        return existed;
+    }
+    async *list() {
+        for (const key of this.index.keys())
+            yield key;
+    }
+    // ── JSON / Blob convenience ───────────────────────────────────────────────
+    async getJSON(key) {
+        const raw = await this.get(key);
+        if (!raw)
+            return undefined;
+        return JSON.parse(DECODE.decode(raw));
+    }
+    async setJSON(key, value) {
+        await this.set(key, TEXT.encode(JSON.stringify(value)));
+    }
+    async getBlob(key) {
+        const raw = await this.get(key);
+        if (!raw)
+            return undefined;
+        return DECODE.decode(raw);
+    }
+    async setBlob(key, content) {
+        await this.set(key, TEXT.encode(content));
+    }
+    // ── Unified entry log ─────────────────────────────────────────────────────
+    async appendEntry(entry) {
+        const seqKey = "meta/log_entry_seq";
+        const current = (await this.getJSON(seqKey)) ?? this.entryCounter;
+        const id = current + 1;
+        this.entryCounter = id;
+        await this.setJSON(seqKey, id);
+        const full = { ...entry, id, ts: Date.now() };
+        const logKey = `core/log/${String(id).padStart(12, "0")}`;
+        await this.setJSON(logKey, full);
+        return full;
+    }
+    async getEntries(filter) {
+        const results = [];
+        for (const key of this.index.keys()) {
+            if (!key.startsWith("core/log/") || key === "core/log/__seq")
+                continue;
+            const entry = await this.getJSON(key);
+            if (!entry)
+                continue;
+            if (filter?.kind !== undefined && entry.kind !== filter.kind)
+                continue;
+            if (filter?.content_source !== undefined && entry.content_source !== filter.content_source)
+                continue;
+            if (filter?.keyPrefix && !entry.key.startsWith(filter.keyPrefix))
+                continue;
+            if (filter?.since !== undefined && entry.ts < filter.since)
+                continue;
+            if (!filter?.includeDeleted && (entry.flags ?? 0) & 0x01)
+                continue;
+            results.push(entry);
+        }
+        return results.sort((a, b) => a.id - b.id);
+    }
+    async getEntry(key) {
+        return this.getJSON(key);
+    }
+    // ── Topology ──────────────────────────────────────────────────────────────
+    async getTopology(kind) {
+        return (await this.getJSON(`topology/${kind}`)) ?? [];
+    }
+    async setTopology(kind, entries) {
+        await this.setJSON(`topology/${kind}`, entries);
+    }
+    // ── Knowledge: agents ─────────────────────────────────────────────────────
+    async getAgentInstruction(agent, file) {
+        return this.getBlob(`knowledge/agents/${agent}/${file}`);
+    }
+    async setAgentInstruction(agent, file, content, source = ContentSource.Package) {
+        const key = `knowledge/agents/${agent}/${file}`;
+        await this.setBlob(key, content);
+        await this.appendEntry({
+            kind: EntityKind.AgentInstruction,
+            content_source: source,
+            key,
+            payload: { agent, file, length: content.length },
+        });
+    }
+    async listAgents() {
+        const agents = new Set();
+        for (const key of this.index.keys()) {
+            if (key.startsWith("knowledge/agents/")) {
+                const parts = key.split("/");
+                if (parts[2])
+                    agents.add(parts[2]);
+            }
+        }
+        return [...agents].sort();
+    }
+    // ── Knowledge: skills ─────────────────────────────────────────────────────
+    async getSkillContent(skill, file) {
+        return this.getBlob(`knowledge/skills/${skill}/${file}`);
+    }
+    async setSkillContent(skill, file, content, source = ContentSource.Package) {
+        const key = `knowledge/skills/${skill}/${file}`;
+        await this.setBlob(key, content);
+        await this.appendEntry({
+            kind: EntityKind.SkillDefinition,
+            content_source: source,
+            key,
+            payload: { skill, file, length: content.length },
+        });
+    }
+    async listSkills() {
+        const skills = new Set();
+        for (const key of this.index.keys()) {
+            if (key.startsWith("knowledge/skills/")) {
+                const parts = key.split("/");
+                if (parts[2])
+                    skills.add(parts[2]);
+            }
+        }
+        return [...skills].sort();
+    }
+    // ── Internal helpers ──────────────────────────────────────────────────────
+    /** Dead space ratio — used to decide if compaction is needed */
+    get deadSpaceRatio() {
+        if (!existsSync(this.storePath))
+            return 0;
+        const totalSize = this.chunkEnd;
+        const liveSize = HEADER_SIZE + [...this.index.values()].reduce((s, e) => s + 4 + e.length, 0);
+        return totalSize > 0 ? 1 - liveSize / totalSize : 0;
+    }
+    /** Number of entries in the unified log */
+    get entryCount() {
+        return this.entryCounter;
+    }
+}
+// ── Factory ────────────────────────────────────────────────────────────────────
+export async function openStore(storePath, opts = {}) {
+    const store = new AcePackedStore();
+    await store.open(storePath, opts);
+    return store;
+}
+//# sourceMappingURL=ace-packed-store.js.map

package/dist/store/bootstrap-store.d.ts

@@ -0,0 +1,46 @@
+/**
+ * bootstrap-store.ts — Phase 8
+ *
+ * New bootstrapAceWorkspace() using the Zarrita-backed store.
+ *
+ * Replaces the old copyTree()-based bootstrap with:
+ *   1. Create minimal directories
+ *   2. Initialize AcePackedStore at .agents/ACE/ace-state.zarr
+ *   3. Bake core knowledge (agents, modules, kernel)
+ *   4. Write topology records
+ *   5. Write initial runtime state seeds
+ *   6. Write meta (codebooks, schema version)
+ *   7. Compact the store
+ *   8. Optionally materialize host integration files
+ *   9. Materialize hook context snapshot
+ *  10. Write tasks/todo.md starter
+ */
+export interface BootstrapOptions {
+    workspaceRoot: string;
+    projectName?: string;
+    force?: boolean;
+    includeMcpConfig?: boolean;
+    includeClientConfigBundle?: boolean;
+    mcpServerPath?: string;
+    llm?: string;
+    model?: string;
+    ollamaUrl?: string;
+}
+export interface BootstrapResult {
+    storePath: string;
+    agents: number;
+    modules: {
+        gates: number;
+        roles: number;
+        schemas: number;
+    };
+    materialized: string[];
+    warnings: string[];
+}
+export declare function bootstrapStoreWorkspace(opts: BootstrapOptions): Promise<BootstrapResult>;
+export declare function repairWorkspace(workspaceRoot: string): Promise<string[]>;
+export declare function compactWorkspace(workspaceRoot: string): Promise<{
+    before: number;
+    after: number;
+}>;
+//# sourceMappingURL=bootstrap-store.d.ts.map