@psiclawops/hypermem 0.5.0 → 0.5.2

package/dist/doc-chunk-store.js

@@ -0,0 +1,391 @@
+/**
+ * hypermem Document Chunk Store
+ *
+ * Manages doc_chunks in library.db:
+ * - Atomic re-indexing by source hash (no stale/fresh coexistence)
+ * - FTS5 keyword search fallback
+ * - Collection-scoped queries with agent/tier filtering
+ * - Source tracking (what's indexed, when, what hash)
+ */
+// ─── Store ──────────────────────────────────────────────────────
+export class DocChunkStore {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    /**
+     * Index a set of chunks for a source file.
+     *
+     * Atomic re-indexing:
+     * 1. Check if source_hash has changed
+     * 2. If unchanged: skip (idempotent)
+     * 3. If changed: delete all chunks with old hash, insert new chunks — in one transaction
+     *
+     * This ensures no window where stale and fresh chunks coexist.
+     */
+    indexChunks(chunks) {
+        if (chunks.length === 0) {
+            return { inserted: 0, deleted: 0, reindexed: false, skipped: true };
+        }
+        const first = chunks[0];
+        const { sourcePath, collection, sourceHash, scope, agentId } = first;
+        const now = new Date().toISOString();
+        // Check current indexed state
+        const existing = this.db
+            .prepare('SELECT source_hash, chunk_count FROM doc_sources WHERE source_path = ? AND collection = ?')
+            .get(sourcePath, collection);
+        if (existing && existing.source_hash === sourceHash) {
+            // Hash unchanged — no-op
+            return { inserted: 0, deleted: 0, reindexed: false, skipped: true };
+        }
+        // Hash changed (or first index) — atomic swap
+        let deleted = 0;
+        let inserted = 0;
+        // Use a transaction for atomicity
+        const run = this.db.prepare('SELECT 1').get; // warm
+        try {
+            // Begin transaction via exec
+            this.db.exec('BEGIN');
+            // Delete stale chunks for this source
+            if (existing) {
+                const result = this.db
+                    .prepare('DELETE FROM doc_chunks WHERE source_path = ? AND collection = ?')
+                    .run(sourcePath, collection);
+                deleted = result.changes;
+            }
+            // Insert new chunks
+            const insertChunk = this.db.prepare(`
+        INSERT OR REPLACE INTO doc_chunks
+          (id, collection, section_path, depth, content, token_estimate,
+           source_hash, source_path, scope, tier, agent_id, parent_path,
+           created_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+            for (const chunk of chunks) {
+                insertChunk.run(chunk.id, chunk.collection, chunk.sectionPath, chunk.depth, chunk.content, chunk.tokenEstimate, chunk.sourceHash, chunk.sourcePath, chunk.scope, chunk.tier ?? null, chunk.agentId ?? null, chunk.parentPath ?? null, now, now);
+                inserted++;
+            }
+            // Update source tracking
+            this.db.prepare(`
+        INSERT OR REPLACE INTO doc_sources
+          (source_path, collection, scope, agent_id, source_hash, chunk_count, indexed_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+      `).run(sourcePath, collection, scope, agentId ?? null, sourceHash, inserted, now);
+            this.db.exec('COMMIT');
+        }
+        catch (err) {
+            this.db.exec('ROLLBACK');
+            throw err;
+        }
+        return { inserted, deleted, reindexed: !!existing, skipped: false };
+    }
+    /**
+     * Query chunks by collection with optional filters.
+     * Falls back to FTS5 keyword search when keyword is provided.
+     */
+    queryChunks(query) {
+        const { collection, scope, agentId, tier, limit = 20, keyword } = query;
+        if (keyword) {
+            return this.keywordSearch(keyword, query);
+        }
+        // Build WHERE clause
+        const conditions = ['collection = ?'];
+        const params = [collection];
+        if (scope) {
+            conditions.push('scope = ?');
+            params.push(scope);
+        }
+        if (agentId) {
+            conditions.push('(agent_id = ? OR agent_id IS NULL)');
+            params.push(agentId);
+        }
+        if (tier) {
+            conditions.push('(tier = ? OR tier IS NULL OR tier = \'all\')');
+            params.push(tier);
+        }
+        params.push(limit);
+        const rows = this.db
+            .prepare(`
+        SELECT id, collection, section_path, depth, content, token_estimate,
+               source_hash, source_path, scope, tier, agent_id, parent_path,
+               created_at, updated_at
+        FROM doc_chunks
+        WHERE ${conditions.join(' AND ')}
+        ORDER BY depth ASC, section_path ASC
+        LIMIT ?
+      `)
+            .all(...params);
+        return rows.map(this.mapRow);
+    }
+    /**
+     * FTS5 keyword search across chunks.
+     */
+    keywordSearch(keyword, query) {
+        const { collection, agentId, tier, limit = 20 } = query;
+        const hasFilters = !!(agentId || tier);
+        const innerLimit = hasFilters ? limit * 4 : limit;
+        // Two-phase: FTS in subquery, metadata filter on small result set.
+        let sql = `
+      SELECT c.id, c.collection, c.section_path, c.depth, c.content, c.token_estimate,
+             c.source_hash, c.source_path, c.scope, c.tier, c.agent_id, c.parent_path,
+             c.created_at, c.updated_at
+      FROM (
+        SELECT rowid, rank FROM doc_chunks_fts WHERE doc_chunks_fts MATCH ? ORDER BY rank LIMIT ?
+      ) sub
+      JOIN doc_chunks c ON c.rowid = sub.rowid
+      WHERE c.collection = ?
+    `;
+        const params = [keyword, innerLimit, collection];
+        if (agentId) {
+            sql += ' AND (c.agent_id = ? OR c.agent_id IS NULL)';
+            params.push(agentId);
+        }
+        if (tier) {
+            sql += " AND (c.tier = ? OR c.tier IS NULL OR c.tier = 'all')";
+            params.push(tier);
+        }
+        sql += ' ORDER BY sub.rank LIMIT ?';
+        params.push(limit * 3); // over-fetch to allow dedup
+        const rows = this.db.prepare(sql).all(...params);
+        // Deduplicate by source_hash to avoid returning identical content
+        // from multiple agent-specific copies of shared-fleet docs.
+        const seenHashes = new Set();
+        const deduped = rows.filter(r => {
+            const hash = r['source_hash'];
+            if (!hash)
+                return true;
+            if (seenHashes.has(hash))
+                return false;
+            seenHashes.add(hash);
+            return true;
+        });
+        return deduped.slice(0, limit).map(this.mapRow);
+    }
+    /**
+     * Get a single chunk by ID.
+     */
+    getChunk(id) {
+        const row = this.db
+            .prepare(`
+        SELECT id, collection, section_path, depth, content, token_estimate,
+               source_hash, source_path, scope, tier, agent_id, parent_path,
+               created_at, updated_at
+        FROM doc_chunks WHERE id = ?
+      `)
+            .get(id);
+        return row ? this.mapRow(row) : null;
+    }
+    /**
+     * Check if a source file needs re-indexing.
+     * Returns true if the file has changed or has never been indexed.
+     */
+    needsReindex(sourcePath, collection, currentHash) {
+        const row = this.db
+            .prepare('SELECT source_hash FROM doc_sources WHERE source_path = ? AND collection = ?')
+            .get(sourcePath, collection);
+        return !row || row.source_hash !== currentHash;
+    }
+    /**
+     * List all indexed sources, optionally filtered by agent or collection.
+     */
+    listSources(opts) {
+        const conditions = [];
+        const params = [];
+        if (opts?.agentId) {
+            conditions.push('agent_id = ?');
+            params.push(opts.agentId);
+        }
+        if (opts?.collection) {
+            conditions.push('collection = ?');
+            params.push(opts.collection);
+        }
+        const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+        const rows = this.db
+            .prepare(`
+        SELECT source_path, collection, scope, agent_id, source_hash, chunk_count, indexed_at
+        FROM doc_sources ${where}
+        ORDER BY indexed_at DESC
+      `)
+            .all(...params);
+        return rows.map(r => ({
+            sourcePath: r['source_path'],
+            collection: r['collection'],
+            scope: r['scope'],
+            agentId: r['agent_id'],
+            sourceHash: r['source_hash'],
+            chunkCount: r['chunk_count'],
+            indexedAt: r['indexed_at'],
+        }));
+    }
+    /**
+     * Delete all chunks for a specific source file.
+     */
+    deleteSource(sourcePath, collection) {
+        this.db.exec('BEGIN');
+        try {
+            const result = this.db
+                .prepare('DELETE FROM doc_chunks WHERE source_path = ? AND collection = ?')
+                .run(sourcePath, collection);
+            this.db.prepare('DELETE FROM doc_sources WHERE source_path = ? AND collection = ?')
+                .run(sourcePath, collection);
+            this.db.exec('COMMIT');
+            return result.changes;
+        }
+        catch (err) {
+            this.db.exec('ROLLBACK');
+            throw err;
+        }
+    }
+    /**
+     * Index simple string chunks with an optional session key (for ephemeral spawn context).
+     *
+     * Unlike indexChunks() which works with DocChunk objects and hash-based dedup,
+     * this method is designed for ad-hoc session-scoped content: it always inserts fresh
+     * rows tagged with the sessionKey, without hash-based skip logic.
+     *
+     * Chunks stored with a sessionKey are ephemeral — use clearSessionChunks() to remove them.
+     */
+    indexDocChunks(agentId, source, chunks, options) {
+        if (chunks.length === 0)
+            return;
+        const now = new Date().toISOString();
+        const sessionKey = options?.sessionKey ?? null;
+        // Use a stable collection name derived from source path
+        const collection = `spawn/${agentId}`;
+        try {
+            this.db.exec('BEGIN');
+            const insert = this.db.prepare(`
+        INSERT INTO doc_chunks
+          (id, collection, section_path, depth, content, token_estimate,
+           source_hash, source_path, scope, tier, agent_id, parent_path,
+           session_key, created_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+            chunks.forEach((chunkContent, idx) => {
+                const id = `spawn:${agentId}:${sessionKey ?? 'none'}:${source}:${idx}:${Date.now()}`;
+                const tokenEstimate = Math.ceil(chunkContent.length / 4);
+                insert.run(id, collection, `${source}#chunk-${idx}`, 2, chunkContent, tokenEstimate, `spawn-${Date.now()}-${idx}`, // non-deduped hash
+                source, 'per-agent', null, agentId, null, sessionKey, now, now);
+            });
+            this.db.exec('COMMIT');
+        }
+        catch (err) {
+            try {
+                this.db.exec('ROLLBACK');
+            }
+            catch { /* ignore */ }
+            console.warn('[hypermem:doc-chunk-store] indexDocChunks failed:', err.message);
+        }
+    }
+    /**
+     * Query doc chunks by agentId+query string, with optional session key scoping.
+     * When sessionKey is provided, only chunks tagged with that session key are returned.
+     */
+    queryDocChunks(agentId, query, options) {
+        const limit = options?.limit ?? 10;
+        const sessionKey = options?.sessionKey;
+        const collection = `spawn/${agentId}`;
+        try {
+            if (query.trim() && query.trim().length >= 3) {
+                // FTS5 keyword search
+                let sql = `
+          SELECT c.id, c.collection, c.section_path, c.depth, c.content, c.token_estimate,
+                 c.source_hash, c.source_path, c.scope, c.tier, c.agent_id, c.parent_path,
+                 c.created_at, c.updated_at
+          FROM (
+            SELECT rowid, rank FROM doc_chunks_fts WHERE doc_chunks_fts MATCH ? ORDER BY rank LIMIT ?
+          ) sub
+          JOIN doc_chunks c ON c.rowid = sub.rowid
+          WHERE c.collection = ?
+        `;
+                const params = [query, limit * 3, collection];
+                if (sessionKey !== undefined) {
+                    sql += ' AND c.session_key = ?';
+                    params.push(sessionKey);
+                }
+                sql += ' ORDER BY sub.rank LIMIT ?';
+                params.push(limit);
+                const rows = this.db.prepare(sql).all(...params);
+                return rows.map(this.mapRow);
+            }
+            else {
+                // Fallback: return most recent chunks for this session
+                let sql = `
+          SELECT id, collection, section_path, depth, content, token_estimate,
+                 source_hash, source_path, scope, tier, agent_id, parent_path,
+                 created_at, updated_at
+          FROM doc_chunks
+          WHERE collection = ?
+        `;
+                const params = [collection];
+                if (sessionKey !== undefined) {
+                    sql += ' AND session_key = ?';
+                    params.push(sessionKey);
+                }
+                sql += ' ORDER BY created_at DESC LIMIT ?';
+                params.push(limit);
+                const rows = this.db.prepare(sql).all(...params);
+                return rows.map(this.mapRow);
+            }
+        }
+        catch (err) {
+            console.warn('[hypermem:doc-chunk-store] queryDocChunks failed:', err.message);
+            return [];
+        }
+    }
+    /**
+     * Delete all doc chunks associated with a specific session key.
+     * Call this when a spawn session is complete to release ephemeral storage.
+     */
+    clearSessionChunks(sessionKey) {
+        try {
+            const result = this.db
+                .prepare('DELETE FROM doc_chunks WHERE session_key = ?')
+                .run(sessionKey);
+            return result.changes;
+        }
+        catch (err) {
+            console.warn('[hypermem:doc-chunk-store] clearSessionChunks failed:', err.message);
+            return 0;
+        }
+    }
+    /**
+     * Get chunk stats: count per collection.
+     */
+    getStats() {
+        const rows = this.db.prepare(`
+      SELECT collection,
+             COUNT(*) as count,
+             COUNT(DISTINCT source_path) as sources,
+             SUM(token_estimate) as total_tokens
+      FROM doc_chunks
+      GROUP BY collection
+      ORDER BY collection
+    `).all();
+        return rows.map(r => ({
+            collection: r['collection'],
+            count: r['count'],
+            sources: r['sources'],
+            totalTokens: r['total_tokens'] ?? 0,
+        }));
+    }
+    mapRow(r) {
+        return {
+            id: r['id'],
+            collection: r['collection'],
+            sectionPath: r['section_path'],
+            depth: r['depth'],
+            content: r['content'],
+            tokenEstimate: r['token_estimate'],
+            sourceHash: r['source_hash'],
+            sourcePath: r['source_path'],
+            scope: r['scope'],
+            tier: r['tier'],
+            agentId: r['agent_id'],
+            parentPath: r['parent_path'],
+            createdAt: r['created_at'],
+            updatedAt: r['updated_at'],
+        };
+    }
+}
+//# sourceMappingURL=doc-chunk-store.js.map

package/dist/doc-chunker.d.ts

@@ -0,0 +1,99 @@
+/**
+ * hypermem Document Chunker
+ *
+ * Splits markdown documents into semantically coherent chunks for L3 indexing.
+ *
+ * Design principles:
+ * - Chunk by logical section (## / ###), NOT by token count
+ * - Each chunk is a self-contained policy/operational unit
+ * - Preserve section hierarchy for context assembly
+ * - Track source file hash for atomic re-indexing
+ * - Idempotent: same source produces same chunks (deterministic IDs)
+ *
+ * Collections (as defined in ACA offload spec):
+ *   governance/policy   — POLICY.md, shared-fleet
+ *   governance/charter  — CHARTER.md, per-tier (council/director)
+ *   governance/comms    — COMMS.md, shared-fleet
+ *   operations/agents   — AGENTS.md, per-tier
+ *   operations/tools    — TOOLS.md, per-agent
+ *   memory/decisions    — MEMORY.md, per-agent
+ *   memory/daily        — memory/YYYY-MM-DD.md, per-agent
+ *   identity/soul       — SOUL.md, per-agent (always-loaded kernel, but still indexed)
+ *   identity/job        — JOB.md, per-agent (demand-loaded during deliberation)
+ */
+export interface DocChunk {
+    /** Unique deterministic ID: sha256(collection + sectionPath + sourceHash) */
+    id: string;
+    /** Collection path: governance/policy, operations/tools, etc. */
+    collection: string;
+    /** Full section path: "§3 > Naming > Single-Name Rule" */
+    sectionPath: string;
+    /** Section depth (0=root, 1=#, 2=##, 3=###) */
+    depth: number;
+    /** The actual text content of this chunk */
+    content: string;
+    /** Token estimate (rough: chars / 4) */
+    tokenEstimate: number;
+    /** SHA-256 of the source file at time of chunking */
+    sourceHash: string;
+    /** Source file path (relative to workspace) */
+    sourcePath: string;
+    /** Scope: shared-fleet | per-tier | per-agent */
+    scope: 'shared-fleet' | 'per-tier' | 'per-agent';
+    /** Tier filter (for per-tier scope): council | director | all */
+    tier?: string;
+    /** Agent ID (for per-agent scope) */
+    agentId?: string;
+    /** Parent section path (for hierarchy context) */
+    parentPath?: string;
+}
+export interface ChunkOptions {
+    collection: string;
+    sourcePath: string;
+    scope: DocChunk['scope'];
+    tier?: string;
+    agentId?: string;
+    /** Minimum content length to emit a chunk (avoids empty section headers) */
+    minContentLen?: number;
+    /** Whether to include parent context prefix in chunk content */
+    includeParentContext?: boolean;
+}
+/**
+ * Hash a string with SHA-256.
+ */
+export declare function hashContent(content: string): string;
+/**
+ * Chunk a markdown document into semantic sections.
+ *
+ * Approach:
+ * - Level 1 (#) headings become top-level section anchors
+ * - Level 2 (##) headings become primary chunks
+ * - Level 3 (###) headings become sub-chunks under their parent
+ * - Content before the first heading becomes a "preamble" chunk
+ * - Empty sections (heading only, no content) are skipped unless minContentLen=0
+ *
+ * For documents with deeply nested content, we group level-3 sections under
+ * their parent level-2 section. This keeps related policy sections together.
+ */
+export declare function chunkMarkdown(content: string, opts: ChunkOptions): DocChunk[];
+/**
+ * Chunk a file from disk.
+ */
+export declare function chunkFile(filePath: string, opts: Omit<ChunkOptions, 'sourcePath'>): DocChunk[];
+/**
+ * Standard collection definitions for ACA workspace files.
+ * Maps file names to collection paths and scope metadata.
+ */
+export interface CollectionDef {
+    collection: string;
+    scope: DocChunk['scope'];
+    tier?: string;
+    description: string;
+}
+export declare const ACA_COLLECTIONS: Record<string, CollectionDef>;
+/**
+ * Infer the collection definition for a file based on its name.
+ * Returns undefined if the file is not a known ACA file.
+ */
+export declare function inferCollection(fileName: string, agentId?: string): CollectionDef | undefined;
+//# sourceMappingURL=doc-chunker.d.ts.map

package/dist/doc-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"doc-chunker.d.ts","sourceRoot":"","sources":["../src/doc-chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAOH,MAAM,WAAW,QAAQ;IACvB,6EAA6E;IAC7E,EAAE,EAAE,MAAM,CAAC;IACX,iEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,qDAAqD;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,KAAK,EAAE,cAAc,GAAG,UAAU,GAAG,WAAW,CAAC;IACjD,iEAAiE;IACjE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4EAA4E;IAC5E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gEAAgE;IAChE,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAID;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEnD;AAgFD;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,QAAQ,EAAE,CA+I7E;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,CAAC,GAAG,QAAQ,EAAE,CAG9F;AAID;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CA8CzD,CAAC;AAEF;;;GAGG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAc7F"}