@psiclawops/hypermem 0.8.5 → 0.9.1

package/dist/doc-chunk-store.js

@@ -1,391 +0,0 @@
-/**
- * hypermem Document Chunk Store
- *
- * Manages doc_chunks in library.db:
- * - Atomic re-indexing by source hash (no stale/fresh coexistence)
- * - FTS5 keyword search fallback
- * - Collection-scoped queries with agent/tier filtering
- * - Source tracking (what's indexed, when, what hash)
- */
-// ─── Store ──────────────────────────────────────────────────────
-export class DocChunkStore {
-    db;
-    constructor(db) {
-        this.db = db;
-    }
-    /**
-     * Index a set of chunks for a source file.
-     *
-     * Atomic re-indexing:
-     * 1. Check if source_hash has changed
-     * 2. If unchanged: skip (idempotent)
-     * 3. If changed: delete all chunks with old hash, insert new chunks — in one transaction
-     *
-     * This ensures no window where stale and fresh chunks coexist.
-     */
-    indexChunks(chunks) {
-        if (chunks.length === 0) {
-            return { inserted: 0, deleted: 0, reindexed: false, skipped: true };
-        }
-        const first = chunks[0];
-        const { sourcePath, collection, sourceHash, scope, agentId } = first;
-        const now = new Date().toISOString();
-        // Check current indexed state
-        const existing = this.db
-            .prepare('SELECT source_hash, chunk_count FROM doc_sources WHERE source_path = ? AND collection = ?')
-            .get(sourcePath, collection);
-        if (existing && existing.source_hash === sourceHash) {
-            // Hash unchanged — no-op
-            return { inserted: 0, deleted: 0, reindexed: false, skipped: true };
-        }
-        // Hash changed (or first index) — atomic swap
-        let deleted = 0;
-        let inserted = 0;
-        // Use a transaction for atomicity
-        const run = this.db.prepare('SELECT 1').get; // warm
-        try {
-            // Begin transaction via exec
-            this.db.exec('BEGIN');
-            // Delete stale chunks for this source
-            if (existing) {
-                const result = this.db
-                    .prepare('DELETE FROM doc_chunks WHERE source_path = ? AND collection = ?')
-                    .run(sourcePath, collection);
-                deleted = result.changes;
-            }
-            // Insert new chunks
-            const insertChunk = this.db.prepare(`
-        INSERT OR REPLACE INTO doc_chunks
-          (id, collection, section_path, depth, content, token_estimate,
-           source_hash, source_path, scope, tier, agent_id, parent_path,
-           created_at, updated_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-      `);
-            for (const chunk of chunks) {
-                insertChunk.run(chunk.id, chunk.collection, chunk.sectionPath, chunk.depth, chunk.content, chunk.tokenEstimate, chunk.sourceHash, chunk.sourcePath, chunk.scope, chunk.tier ?? null, chunk.agentId ?? null, chunk.parentPath ?? null, now, now);
-                inserted++;
-            }
-            // Update source tracking
-            this.db.prepare(`
-        INSERT OR REPLACE INTO doc_sources
-          (source_path, collection, scope, agent_id, source_hash, chunk_count, indexed_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?)
-      `).run(sourcePath, collection, scope, agentId ?? null, sourceHash, inserted, now);
-            this.db.exec('COMMIT');
-        }
-        catch (err) {
-            this.db.exec('ROLLBACK');
-            throw err;
-        }
-        return { inserted, deleted, reindexed: !!existing, skipped: false };
-    }
-    /**
-     * Query chunks by collection with optional filters.
-     * Falls back to FTS5 keyword search when keyword is provided.
-     */
-    queryChunks(query) {
-        const { collection, scope, agentId, tier, limit = 20, keyword } = query;
-        if (keyword) {
-            return this.keywordSearch(keyword, query);
-        }
-        // Build WHERE clause
-        const conditions = ['collection = ?'];
-        const params = [collection];
-        if (scope) {
-            conditions.push('scope = ?');
-            params.push(scope);
-        }
-        if (agentId) {
-            conditions.push('(agent_id = ? OR agent_id IS NULL)');
-            params.push(agentId);
-        }
-        if (tier) {
-            conditions.push('(tier = ? OR tier IS NULL OR tier = \'all\')');
-            params.push(tier);
-        }
-        params.push(limit);
-        const rows = this.db
-            .prepare(`
-        SELECT id, collection, section_path, depth, content, token_estimate,
-               source_hash, source_path, scope, tier, agent_id, parent_path,
-               created_at, updated_at
-        FROM doc_chunks
-        WHERE ${conditions.join(' AND ')}
-        ORDER BY depth ASC, section_path ASC
-        LIMIT ?
-      `)
-            .all(...params);
-        return rows.map(this.mapRow);
-    }
-    /**
-     * FTS5 keyword search across chunks.
-     */
-    keywordSearch(keyword, query) {
-        const { collection, agentId, tier, limit = 20 } = query;
-        const hasFilters = !!(agentId || tier);
-        const innerLimit = hasFilters ? limit * 4 : limit;
-        // Two-phase: FTS in subquery, metadata filter on small result set.
-        let sql = `
-      SELECT c.id, c.collection, c.section_path, c.depth, c.content, c.token_estimate,
-             c.source_hash, c.source_path, c.scope, c.tier, c.agent_id, c.parent_path,
-             c.created_at, c.updated_at
-      FROM (
-        SELECT rowid, rank FROM doc_chunks_fts WHERE doc_chunks_fts MATCH ? ORDER BY rank LIMIT ?
-      ) sub
-      JOIN doc_chunks c ON c.rowid = sub.rowid
-      WHERE c.collection = ?
-    `;
-        const params = [keyword, innerLimit, collection];
-        if (agentId) {
-            sql += ' AND (c.agent_id = ? OR c.agent_id IS NULL)';
-            params.push(agentId);
-        }
-        if (tier) {
-            sql += " AND (c.tier = ? OR c.tier IS NULL OR c.tier = 'all')";
-            params.push(tier);
-        }
-        sql += ' ORDER BY sub.rank LIMIT ?';
-        params.push(limit * 3); // over-fetch to allow dedup
-        const rows = this.db.prepare(sql).all(...params);
-        // Deduplicate by source_hash to avoid returning identical content
-        // from multiple agent-specific copies of shared-fleet docs.
-        const seenHashes = new Set();
-        const deduped = rows.filter(r => {
-            const hash = r['source_hash'];
-            if (!hash)
-                return true;
-            if (seenHashes.has(hash))
-                return false;
-            seenHashes.add(hash);
-            return true;
-        });
-        return deduped.slice(0, limit).map(this.mapRow);
-    }
-    /**
-     * Get a single chunk by ID.
-     */
-    getChunk(id) {
-        const row = this.db
-            .prepare(`
-        SELECT id, collection, section_path, depth, content, token_estimate,
-               source_hash, source_path, scope, tier, agent_id, parent_path,
-               created_at, updated_at
-        FROM doc_chunks WHERE id = ?
-      `)
-            .get(id);
-        return row ? this.mapRow(row) : null;
-    }
-    /**
-     * Check if a source file needs re-indexing.
-     * Returns true if the file has changed or has never been indexed.
-     */
-    needsReindex(sourcePath, collection, currentHash) {
-        const row = this.db
-            .prepare('SELECT source_hash FROM doc_sources WHERE source_path = ? AND collection = ?')
-            .get(sourcePath, collection);
-        return !row || row.source_hash !== currentHash;
-    }
-    /**
-     * List all indexed sources, optionally filtered by agent or collection.
-     */
-    listSources(opts) {
-        const conditions = [];
-        const params = [];
-        if (opts?.agentId) {
-            conditions.push('agent_id = ?');
-            params.push(opts.agentId);
-        }
-        if (opts?.collection) {
-            conditions.push('collection = ?');
-            params.push(opts.collection);
-        }
-        const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
-        const rows = this.db
-            .prepare(`
-        SELECT source_path, collection, scope, agent_id, source_hash, chunk_count, indexed_at
-        FROM doc_sources ${where}
-        ORDER BY indexed_at DESC
-      `)
-            .all(...params);
-        return rows.map(r => ({
-            sourcePath: r['source_path'],
-            collection: r['collection'],
-            scope: r['scope'],
-            agentId: r['agent_id'],
-            sourceHash: r['source_hash'],
-            chunkCount: r['chunk_count'],
-            indexedAt: r['indexed_at'],
-        }));
-    }
-    /**
-     * Delete all chunks for a specific source file.
-     */
-    deleteSource(sourcePath, collection) {
-        this.db.exec('BEGIN');
-        try {
-            const result = this.db
-                .prepare('DELETE FROM doc_chunks WHERE source_path = ? AND collection = ?')
-                .run(sourcePath, collection);
-            this.db.prepare('DELETE FROM doc_sources WHERE source_path = ? AND collection = ?')
-                .run(sourcePath, collection);
-            this.db.exec('COMMIT');
-            return result.changes;
-        }
-        catch (err) {
-            this.db.exec('ROLLBACK');
-            throw err;
-        }
-    }
-    /**
-     * Index simple string chunks with an optional session key (for ephemeral spawn context).
-     *
-     * Unlike indexChunks() which works with DocChunk objects and hash-based dedup,
-     * this method is designed for ad-hoc session-scoped content: it always inserts fresh
-     * rows tagged with the sessionKey, without hash-based skip logic.
-     *
-     * Chunks stored with a sessionKey are ephemeral — use clearSessionChunks() to remove them.
-     */
-    indexDocChunks(agentId, source, chunks, options) {
-        if (chunks.length === 0)
-            return;
-        const now = new Date().toISOString();
-        const sessionKey = options?.sessionKey ?? null;
-        // Use a stable collection name derived from source path
-        const collection = `spawn/${agentId}`;
-        try {
-            this.db.exec('BEGIN');
-            const insert = this.db.prepare(`
-        INSERT INTO doc_chunks
-          (id, collection, section_path, depth, content, token_estimate,
-           source_hash, source_path, scope, tier, agent_id, parent_path,
-           session_key, created_at, updated_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-      `);
-            chunks.forEach((chunkContent, idx) => {
-                const id = `spawn:${agentId}:${sessionKey ?? 'none'}:${source}:${idx}:${Date.now()}`;
-                const tokenEstimate = Math.ceil(chunkContent.length / 4);
-                insert.run(id, collection, `${source}#chunk-${idx}`, 2, chunkContent, tokenEstimate, `spawn-${Date.now()}-${idx}`, // non-deduped hash
-                source, 'per-agent', null, agentId, null, sessionKey, now, now);
-            });
-            this.db.exec('COMMIT');
-        }
-        catch (err) {
-            try {
-                this.db.exec('ROLLBACK');
-            }
-            catch { /* ignore */ }
-            console.warn('[hypermem:doc-chunk-store] indexDocChunks failed:', err.message);
-        }
-    }
-    /**
-     * Query doc chunks by agentId+query string, with optional session key scoping.
-     * When sessionKey is provided, only chunks tagged with that session key are returned.
-     */
-    queryDocChunks(agentId, query, options) {
-        const limit = options?.limit ?? 10;
-        const sessionKey = options?.sessionKey;
-        const collection = `spawn/${agentId}`;
-        try {
-            if (query.trim() && query.trim().length >= 3) {
-                // FTS5 keyword search
-                let sql = `
-          SELECT c.id, c.collection, c.section_path, c.depth, c.content, c.token_estimate,
-                 c.source_hash, c.source_path, c.scope, c.tier, c.agent_id, c.parent_path,
-                 c.created_at, c.updated_at
-          FROM (
-            SELECT rowid, rank FROM doc_chunks_fts WHERE doc_chunks_fts MATCH ? ORDER BY rank LIMIT ?
-          ) sub
-          JOIN doc_chunks c ON c.rowid = sub.rowid
-          WHERE c.collection = ?
-        `;
-                const params = [query, limit * 3, collection];
-                if (sessionKey !== undefined) {
-                    sql += ' AND c.session_key = ?';
-                    params.push(sessionKey);
-                }
-                sql += ' ORDER BY sub.rank LIMIT ?';
-                params.push(limit);
-                const rows = this.db.prepare(sql).all(...params);
-                return rows.map(this.mapRow);
-            }
-            else {
-                // Fallback: return most recent chunks for this session
-                let sql = `
-          SELECT id, collection, section_path, depth, content, token_estimate,
-                 source_hash, source_path, scope, tier, agent_id, parent_path,
-                 created_at, updated_at
-          FROM doc_chunks
-          WHERE collection = ?
-        `;
-                const params = [collection];
-                if (sessionKey !== undefined) {
-                    sql += ' AND session_key = ?';
-                    params.push(sessionKey);
-                }
-                sql += ' ORDER BY created_at DESC LIMIT ?';
-                params.push(limit);
-                const rows = this.db.prepare(sql).all(...params);
-                return rows.map(this.mapRow);
-            }
-        }
-        catch (err) {
-            console.warn('[hypermem:doc-chunk-store] queryDocChunks failed:', err.message);
-            return [];
-        }
-    }
-    /**
-     * Delete all doc chunks associated with a specific session key.
-     * Call this when a spawn session is complete to release ephemeral storage.
-     */
-    clearSessionChunks(sessionKey) {
-        try {
-            const result = this.db
-                .prepare('DELETE FROM doc_chunks WHERE session_key = ?')
-                .run(sessionKey);
-            return result.changes;
-        }
-        catch (err) {
-            console.warn('[hypermem:doc-chunk-store] clearSessionChunks failed:', err.message);
-            return 0;
-        }
-    }
-    /**
-     * Get chunk stats: count per collection.
-     */
-    getStats() {
-        const rows = this.db.prepare(`
-      SELECT collection,
-             COUNT(*) as count,
-             COUNT(DISTINCT source_path) as sources,
-             SUM(token_estimate) as total_tokens
-      FROM doc_chunks
-      GROUP BY collection
-      ORDER BY collection
-    `).all();
-        return rows.map(r => ({
-            collection: r['collection'],
-            count: r['count'],
-            sources: r['sources'],
-            totalTokens: r['total_tokens'] ?? 0,
-        }));
-    }
-    mapRow(r) {
-        return {
-            id: r['id'],
-            collection: r['collection'],
-            sectionPath: r['section_path'],
-            depth: r['depth'],
-            content: r['content'],
-            tokenEstimate: r['token_estimate'],
-            sourceHash: r['source_hash'],
-            sourcePath: r['source_path'],
-            scope: r['scope'],
-            tier: r['tier'],
-            agentId: r['agent_id'],
-            parentPath: r['parent_path'],
-            createdAt: r['created_at'],
-            updatedAt: r['updated_at'],
-        };
-    }
-}
-//# sourceMappingURL=doc-chunk-store.js.map

package/dist/doc-chunker.d.ts

@@ -1,99 +0,0 @@
-/**
- * hypermem Document Chunker
- *
- * Splits markdown documents into semantically coherent chunks for L3 indexing.
- *
- * Design principles:
- * - Chunk by logical section (## / ###), NOT by token count
- * - Each chunk is a self-contained policy/operational unit
- * - Preserve section hierarchy for context assembly
- * - Track source file hash for atomic re-indexing
- * - Idempotent: same source produces same chunks (deterministic IDs)
- *
- * Collections (as defined in ACA offload spec):
- *   governance/policy   — POLICY.md, shared-fleet
- *   governance/charter  — CHARTER.md, per-tier (council/director)
- *   governance/comms    — COMMS.md, shared-fleet
- *   operations/agents   — AGENTS.md, per-tier
- *   operations/tools    — TOOLS.md, per-agent
- *   memory/decisions    — MEMORY.md, per-agent
- *   memory/daily        — memory/YYYY-MM-DD.md, per-agent
- *   identity/soul       — SOUL.md, per-agent (always-loaded kernel, but still indexed)
- *   identity/job        — JOB.md, per-agent (demand-loaded during deliberation)
- */
-export interface DocChunk {
-    /** Unique deterministic ID: sha256(collection + sectionPath + sourceHash) */
-    id: string;
-    /** Collection path: governance/policy, operations/tools, etc. */
-    collection: string;
-    /** Full section path: "§3 > Naming > Single-Name Rule" */
-    sectionPath: string;
-    /** Section depth (0=root, 1=#, 2=##, 3=###) */
-    depth: number;
-    /** The actual text content of this chunk */
-    content: string;
-    /** Token estimate (rough: chars / 4) */
-    tokenEstimate: number;
-    /** SHA-256 of the source file at time of chunking */
-    sourceHash: string;
-    /** Source file path (relative to workspace) */
-    sourcePath: string;
-    /** Scope: shared-fleet | per-tier | per-agent */
-    scope: 'shared-fleet' | 'per-tier' | 'per-agent';
-    /** Tier filter (for per-tier scope): council | director | all */
-    tier?: string;
-    /** Agent ID (for per-agent scope) */
-    agentId?: string;
-    /** Parent section path (for hierarchy context) */
-    parentPath?: string;
-}
-export interface ChunkOptions {
-    collection: string;
-    sourcePath: string;
-    scope: DocChunk['scope'];
-    tier?: string;
-    agentId?: string;
-    /** Minimum content length to emit a chunk (avoids empty section headers) */
-    minContentLen?: number;
-    /** Whether to include parent context prefix in chunk content */
-    includeParentContext?: boolean;
-}
-/**
- * Hash a string with SHA-256.
- */
-export declare function hashContent(content: string): string;
-/**
- * Chunk a markdown document into semantic sections.
- *
- * Approach:
- * - Level 1 (#) headings become top-level section anchors
- * - Level 2 (##) headings become primary chunks
- * - Level 3 (###) headings become sub-chunks under their parent
- * - Content before the first heading becomes a "preamble" chunk
- * - Empty sections (heading only, no content) are skipped unless minContentLen=0
- *
- * For documents with deeply nested content, we group level-3 sections under
- * their parent level-2 section. This keeps related policy sections together.
- */
-export declare function chunkMarkdown(content: string, opts: ChunkOptions): DocChunk[];
-/**
- * Chunk a file from disk.
- */
-export declare function chunkFile(filePath: string, opts: Omit<ChunkOptions, 'sourcePath'>): DocChunk[];
-/**
- * Standard collection definitions for ACA workspace files.
- * Maps file names to collection paths and scope metadata.
- */
-export interface CollectionDef {
-    collection: string;
-    scope: DocChunk['scope'];
-    tier?: string;
-    description: string;
-}
-export declare const ACA_COLLECTIONS: Record<string, CollectionDef>;
-/**
- * Infer the collection definition for a file based on its name.
- * Returns undefined if the file is not a known ACA file.
- */
-export declare function inferCollection(fileName: string, agentId?: string): CollectionDef | undefined;
-//# sourceMappingURL=doc-chunker.d.ts.map

package/dist/doc-chunker.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"doc-chunker.d.ts","sourceRoot":"","sources":["../src/doc-chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAOH,MAAM,WAAW,QAAQ;IACvB,6EAA6E;IAC7E,EAAE,EAAE,MAAM,CAAC;IACX,iEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,qDAAqD;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,KAAK,EAAE,cAAc,GAAG,UAAU,GAAG,WAAW,CAAC;IACjD,iEAAiE;IACjE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4EAA4E;IAC5E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gEAAgE;IAChE,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAID;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEnD;AAgFD;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,QAAQ,EAAE,CA+I7E;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,CAAC,GAAG,QAAQ,EAAE,CAG9F;AAID;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CA8CzD,CAAC;AAEF;;;GAGG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAc7F"}