n2-soul 4.1.0

package/lib/kv-cache/tier-manager.js

@@ -0,0 +1,239 @@
+// Soul KV-Cache — Tiered storage manager. Hot/Warm/Cold lifecycle for snapshots.
+const path = require('path');
+const fs = require('fs');
+
+/**
+ * Tiered storage levels for KV-Cache snapshots.
+ *
+ * Hot  (0-7 days):  In-memory cache + disk. Fastest access.
+ * Warm (8-30 days): Disk only. Normal file/db access.
+ * Cold (30+ days):  Archived (compressed). Lazy load on demand.
+ */
+const TIERS = {
+    HOT: { name: 'hot', maxAgeDays: 7 },
+    WARM: { name: 'warm', maxAgeDays: 30 },
+    COLD: { name: 'cold', maxAgeDays: Infinity },
+};
+
+/**
+ * TierManager wraps a storage engine and adds tiered caching.
+ * Hot tier snapshots are kept in memory for fast access.
+ * Cold tier snapshots are moved to a compressed archive.
+ */
+class TierManager {
+    /**
+     * @param {object} storageEngine - SnapshotEngine or SqliteStore
+     * @param {object} config - tier config { hotDays, warmDays }
+     */
+    constructor(storageEngine, config = {}) {
+        this.engine = storageEngine;
+        this.hotDays = config.hotDays || TIERS.HOT.maxAgeDays;
+        this.warmDays = config.warmDays || TIERS.WARM.maxAgeDays;
+        this._hotCache = {};  // { projectName: { id: session } }
+    }
+
+    /**
+     * Classify a snapshot's tier based on age.
+     *
+     * @param {object} snapshot
+     * @returns {'hot'|'warm'|'cold'}
+     */
+    classify(snapshot) {
+        const timestamp = snapshot.endedAt || snapshot.startedAt;
+        if (!timestamp) return 'warm';
+
+        const ageMs = Date.now() - new Date(timestamp).getTime();
+        const ageDays = ageMs / (24 * 60 * 60 * 1000);
+
+        if (ageDays <= this.hotDays) return 'hot';
+        if (ageDays <= this.warmDays) return 'warm';
+        return 'cold';
+    }
+
+    /**
+     * Save with automatic hot-cache population.
+     *
+     * @param {object} session
+     * @returns {string} Snapshot ID
+     */
+    save(session) {
+        const id = this.engine.save(session);
+
+        // Add to hot cache
+        const project = session.projectName || session.project;
+        if (project) {
+            if (!this._hotCache[project]) this._hotCache[project] = {};
+            this._hotCache[project][id] = { ...session, id };
+        }
+
+        return id;
+    }
+
+    /**
+     * Load latest with hot-cache check first.
+     *
+     * @param {string} projectName
+     * @returns {object|null}
+     */
+    loadLatest(projectName) {
+        // Check hot cache first
+        const cache = this._hotCache[projectName];
+        if (cache) {
+            const entries = Object.values(cache);
+            if (entries.length > 0) {
+                entries.sort((a, b) => {
+                    const ta = new Date(b.endedAt || b.startedAt).getTime();
+                    const tb = new Date(a.endedAt || a.startedAt).getTime();
+                    return ta - tb;
+                });
+                return entries[0];
+            }
+        }
+
+        // Fall through to storage engine
+        return this.engine.loadLatest(projectName);
+    }
+
+    /**
+     * List snapshots with tier annotations.
+     *
+     * @param {string} projectName
+     * @param {number} limit
+     * @returns {object[]}
+     */
+    list(projectName, limit = 10) {
+        const snapshots = this.engine.list(projectName, limit);
+        return snapshots.map(snap => ({
+            ...snap,
+            _tier: this.classify(snap),
+        }));
+    }
+
+    /**
+     * Search with tier annotations.
+     *
+     * @param {string} query
+     * @param {string} projectName
+     * @param {number} limit
+     * @returns {object[]}
+     */
+    search(query, projectName, limit = 10) {
+        const results = this.engine.search(query, projectName, limit);
+        return results.map(snap => ({
+            ...snap,
+            _tier: this.classify(snap),
+        }));
+    }
+
+    /**
+     * Tier-aware garbage collection.
+     * - Hot: never deleted
+     * - Warm: normal retention
+     * - Cold: archived or deleted based on maxAge
+     *
+     * @param {string} projectName
+     * @param {number} maxAgeDays
+     * @param {number} maxCount
+     * @returns {{ deleted: number, hotCount: number, warmCount: number, coldCount: number }}
+     */
+    gc(projectName, maxAgeDays = 30, maxCount = 50) {
+        const result = this.engine.gc(projectName, maxAgeDays, maxCount);
+
+        // Refresh hot cache
+        this._refreshHotCache(projectName);
+
+        // Count tiers after GC
+        const remaining = this.list(projectName, 9999);
+        const counts = { hot: 0, warm: 0, cold: 0 };
+        for (const snap of remaining) {
+            counts[snap._tier || 'warm']++;
+        }
+
+        return {
+            deleted: result.deleted,
+            hotCount: counts.hot,
+            warmCount: counts.warm,
+            coldCount: counts.cold,
+        };
+    }
+
+    /**
+     * Refresh hot cache for a project from storage.
+     * @param {string} projectName
+     */
+    _refreshHotCache(projectName) {
+        const snaps = this.engine.list(projectName, 100);
+        const cache = {};
+
+        for (const snap of snaps) {
+            if (this.classify(snap) === 'hot') {
+                cache[snap.id] = snap;
+            }
+        }
+
+        this._hotCache[projectName] = cache;
+    }
+
+    /**
+     * Get tier distribution summary for a project.
+     *
+     * @param {string} projectName
+     * @returns {{ hot: number, warm: number, cold: number, total: number }}
+     */
+    tierSummary(projectName) {
+        const snaps = this.list(projectName, 9999);
+        const counts = { hot: 0, warm: 0, cold: 0, total: snaps.length };
+
+        for (const snap of snaps) {
+            counts[snap._tier || 'warm']++;
+        }
+
+        return counts;
+    }
+
+    /**
+     * Warm up: preload hot tier snapshots into memory.
+     *
+     * @param {string} projectName
+     * @returns {number} Number of snapshots cached
+     */
+    warmUp(projectName) {
+        this._refreshHotCache(projectName);
+        return Object.keys(this._hotCache[projectName] || {}).length;
+    }
+
+    /**
+     * Clear hot cache for a project.
+     * @param {string} projectName
+     */
+    evict(projectName) {
+        delete this._hotCache[projectName];
+    }
+
+    /**
+     * Proxy: loadById
+     */
+    loadById(projectName, snapshotId) {
+        return this.engine.loadById(projectName, snapshotId);
+    }
+
+    /**
+     * Proxy: migrateFromJson (if available)
+     */
+    migrateFromJson(jsonBaseDir, projectName) {
+        if (this.engine.migrateFromJson) {
+            return this.engine.migrateFromJson(jsonBaseDir, projectName);
+        }
+        return { error: 'Migration not available for this backend' };
+    }
+
+    /**
+     * Proxy: dispose
+     */
+    dispose() {
+        this._hotCache = {};
+        if (this.engine.dispose) this.engine.dispose();
+    }
+}
+
+module.exports = { TierManager, TIERS };

package/lib/kv-cache/token-saver.js

@@ -0,0 +1,153 @@
+// Soul KV-Cache — Progressive loading. Extracts context at L1/L2/L3 token budgets.
+const { extractKeywords } = require('./agent-adapter');
+
+/**
+ * Progressive loading levels for KV-Cache context restoration.
+ * Lower levels use fewer tokens but provide less context.
+ *
+ * L1: Minimal — keywords + TODO only (~500 tokens)
+ * L2: Standard — L1 + compressed summary + decisions (~2000 tokens)
+ * L3: Full — complete uncompressed context (no limit)
+ */
+const LEVELS = {
+    L1: { name: 'minimal', maxTokens: 500 },
+    L2: { name: 'standard', maxTokens: 2000 },
+    L3: { name: 'full', maxTokens: Infinity },
+};
+
+/**
+ * Extracts context from a snapshot at the specified progressive level.
+ *
+ * @param {object} snapshot - KV-Cache session snapshot
+ * @param {string} level - 'L1', 'L2', or 'L3'
+ * @returns {{ level: string, tokens: number, prompt: string }}
+ */
+function extractAtLevel(snapshot, level = 'L2') {
+    if (!snapshot) return { level, tokens: 0, prompt: '' };
+
+    const spec = LEVELS[level] || LEVELS.L2;
+    const lines = [];
+
+    // --- L1: Minimal (keywords + TODO) ---
+    lines.push(`[Session: ${snapshot.agentName} | ${(snapshot.endedAt || snapshot.startedAt || '').split('T')[0]}]`);
+
+    if (snapshot.parentSessionId) {
+        lines.push(`Chain: ${snapshot.parentSessionId.slice(0, 8)} -> ${snapshot.id.slice(0, 8)}`);
+    }
+
+    if (snapshot.keys?.length > 0) {
+        lines.push(`Topics: ${snapshot.keys.slice(0, 10).join(', ')}`);
+    }
+
+    if (snapshot.context?.todo?.length > 0) {
+        lines.push('TODO:');
+        for (const t of snapshot.context.todo) {
+            lines.push(`  - ${t}`);
+        }
+    }
+
+    let prompt = lines.join('\n');
+    let tokens = estimateTokenCount(prompt);
+
+    if (level === 'L1' || tokens >= spec.maxTokens) {
+        return trimToTokenBudget(prompt, tokens, spec.maxTokens, 'L1');
+    }
+
+    // --- L2: Standard (+ summary + decisions) ---
+    if (snapshot.context?.decisions?.length > 0) {
+        lines.push('Decisions:');
+        for (const d of snapshot.context.decisions) {
+            lines.push(`  - ${d}`);
+        }
+    }
+
+    if (snapshot.context?.summary) {
+        lines.push(`Summary: ${snapshot.context.summary}`);
+    }
+
+    prompt = lines.join('\n');
+    tokens = estimateTokenCount(prompt);
+
+    if (level === 'L2' || tokens >= spec.maxTokens) {
+        return trimToTokenBudget(prompt, tokens, spec.maxTokens, 'L2');
+    }
+
+    // --- L3: Full (+ files changed) ---
+    if (snapshot.context?.filesChanged?.length > 0) {
+        lines.push('Files changed:');
+        for (const f of snapshot.context.filesChanged) {
+            const entry = typeof f === 'string' ? f : `${f.path} — ${f.desc || ''}`;
+            lines.push(`  - ${entry}`);
+        }
+    }
+
+    // Include raw metadata
+    lines.push(`Agent type: ${snapshot.agentType || 'unknown'}`);
+    if (snapshot.model) lines.push(`Model: ${snapshot.model}`);
+    if (snapshot.turnCount) lines.push(`Turns: ${snapshot.turnCount}`);
+    if (snapshot.tokenEstimate) lines.push(`Token estimate: ${snapshot.tokenEstimate}`);
+
+    prompt = lines.join('\n');
+    tokens = estimateTokenCount(prompt);
+
+    return { level: 'L3', tokens, prompt };
+}
+
+/**
+ * Auto-selects the best progressive level based on available token budget.
+ * Starts from L3 and downgrades until it fits.
+ *
+ * @param {object} snapshot - KV-Cache session snapshot
+ * @param {number} budgetTokens - Available token budget
+ * @returns {{ level: string, tokens: number, prompt: string }}
+ */
+function autoLevel(snapshot, budgetTokens = 2000) {
+    if (!snapshot) return { level: 'L1', tokens: 0, prompt: '' };
+
+    // Try from highest to lowest
+    for (const lvl of ['L3', 'L2', 'L1']) {
+        const result = extractAtLevel(snapshot, lvl);
+        if (result.tokens <= budgetTokens) {
+            return result;
+        }
+    }
+
+    // Even L1 is over budget — trim L1
+    const l1 = extractAtLevel(snapshot, 'L1');
+    return trimToTokenBudget(l1.prompt, l1.tokens, budgetTokens, 'L1');
+}
+
+/**
+ * Estimates token count for text (model-agnostic).
+ * CJK characters ~1 token each, ASCII ~4 chars per token.
+ *
+ * @param {string} text
+ * @returns {number}
+ */
+function estimateTokenCount(text) {
+    if (!text) return 0;
+    const cjkCount = (text.match(/[\u3000-\u9fff\uac00-\ud7af]/g) || []).length;
+    const asciiCount = text.length - cjkCount;
+    return Math.ceil(asciiCount / 4 + cjkCount / 2);
+}
+
+/**
+ * Trims prompt text to fit within a token budget.
+ *
+ * @param {string} prompt
+ * @param {number} currentTokens
+ * @param {number} maxTokens
+ * @param {string} level
+ * @returns {{ level: string, tokens: number, prompt: string }}
+ */
+function trimToTokenBudget(prompt, currentTokens, maxTokens, level) {
+    if (currentTokens <= maxTokens) {
+        return { level, tokens: currentTokens, prompt };
+    }
+    // Conservative: assume 3 chars per token for trimming
+    const maxChars = maxTokens * 3;
+    const trimmed = prompt.slice(0, maxChars) + '\n...(truncated)';
+    return { level, tokens: maxTokens, prompt: trimmed };
+}
+
+module.exports = { extractAtLevel, autoLevel, estimateTokenCount, LEVELS };

package/lib/paths.js

@@ -0,0 +1,20 @@
+// Soul — Central path manager. Cross-platform compatible.
+const path = require('path');
+const fs = require('fs');
+
+// soul/lib/paths.js → 2 levels up = project root
+const PROJECT_ROOT = path.resolve(__dirname, '..', '..');
+const DATA_ROOT = path.join(path.resolve(__dirname, '..'), 'data');
+
+/** Agents directory path (auto-created if needed) */
+function getAgentsDir() {
+    const dir = path.join(DATA_ROOT, 'agents');
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    return dir;
+}
+
+module.exports = {
+    PROJECT_ROOT,
+    DATA_ROOT,
+    getAgentsDir,
+};

package/lib/soul-engine.js

@@ -0,0 +1,189 @@
+// Soul MCP v4.0 — Soul engine: Board, Ledger, and File Index management.
+const fs = require('fs');
+const path = require('path');
+const { readJson, writeJson, nowISO, logError } = require('./utils');
+
+class SoulEngine {
+    constructor(dataDir) {
+        this.dataDir = dataDir;
+    }
+
+    // -- Path helpers --
+
+    projectDir(projectName) {
+        return path.join(this.dataDir, 'projects', projectName);
+    }
+
+    boardPath(projectName) {
+        return path.join(this.projectDir(projectName), 'soul-board.json');
+    }
+
+    fileIndexPath(projectName) {
+        return path.join(this.projectDir(projectName), 'file-index.json');
+    }
+
+    ledgerDir(projectName, date) {
+        const [y, m, d] = (date || nowISO().split('T')[0]).split('-');
+        return path.join(this.projectDir(projectName), 'ledger', y, m, d);
+    }
+
+    // -- Soul Board --
+
+    readBoard(projectName) {
+        return readJson(this.boardPath(projectName)) || this._defaultBoard(projectName);
+    }
+
+    writeBoard(projectName, board) {
+        board.updatedAt = nowISO();
+        writeJson(this.boardPath(projectName), board);
+    }
+
+    _defaultBoard(projectName) {
+        return {
+            project: projectName,
+            updatedAt: nowISO(),
+            updatedBy: null,
+            state: { summary: '', version: '', health: 'unknown' },
+            activeWork: {},
+            fileOwnership: {},
+            decisions: [],
+            handoff: { from: null, summary: '', todo: [], blockers: [] },
+            lastLedger: null,
+        };
+    }
+
+    // -- File Ownership --
+
+    claimFile(projectName, filePath, agent, intent) {
+        const board = this.readBoard(projectName);
+        const existing = board.fileOwnership[filePath];
+        if (existing && existing.owner && existing.owner !== agent) {
+            return { ok: false, owner: existing.owner, intent: existing.intent };
+        }
+        board.fileOwnership[filePath] = { owner: agent, since: nowISO(), intent };
+        board.updatedBy = agent;
+        this.writeBoard(projectName, board);
+        return { ok: true };
+    }
+
+    releaseFiles(projectName, agent) {
+        const board = this.readBoard(projectName);
+        for (const [fp, info] of Object.entries(board.fileOwnership)) {
+            if (info.owner === agent) {
+                board.fileOwnership[fp] = { owner: null };
+            }
+        }
+        board.updatedBy = agent;
+        this.writeBoard(projectName, board);
+    }
+
+    // -- Active Work --
+
+    setActiveWork(projectName, agent, task, files) {
+        const board = this.readBoard(projectName);
+        board.activeWork[agent] = { task, since: nowISO(), files: files || [] };
+        board.updatedBy = agent;
+        this.writeBoard(projectName, board);
+    }
+
+    clearActiveWork(projectName, agent) {
+        const board = this.readBoard(projectName);
+        board.activeWork[agent] = null;
+        board.updatedBy = agent;
+        this.writeBoard(projectName, board);
+    }
+
+    // -- Ledger --
+
+    getNextLedgerId(projectName, date) {
+        const dir = this.ledgerDir(projectName, date);
+        if (!fs.existsSync(dir)) return '001';
+        const files = fs.readdirSync(dir).filter(f => f.endsWith('.json'));
+        const nums = files.map(f => parseInt(f.split('-')[0]) || 0);
+        const max = nums.length > 0 ? Math.max(...nums) : 0;
+        return String(max + 1).padStart(3, '0');
+    }
+
+    writeLedger(projectName, agent, entry) {
+        const date = nowISO().split('T')[0];
+        const id = this.getNextLedgerId(projectName, date);
+        const ledgerEntry = {
+            id,
+            agent,
+            startedAt: entry.startedAt || nowISO(),
+            completedAt: nowISO(),
+            title: entry.title || 'Untitled work',
+            filesCreated: entry.filesCreated || [],
+            filesModified: entry.filesModified || [],
+            filesDeleted: entry.filesDeleted || [],
+            decisions: entry.decisions || [],
+            summary: entry.summary || '',
+        };
+
+        const dir = this.ledgerDir(projectName, date);
+        const fileName = `${id}-${agent.toLowerCase().replace(/[^a-z0-9]/g, '')}.json`;
+        writeJson(path.join(dir, fileName), ledgerEntry);
+
+        // Update board's lastLedger reference
+        const [y, m, d] = date.split('-');
+        const board = this.readBoard(projectName);
+        board.lastLedger = `${y}/${m}/${d}/${id}-${agent}`;
+        board.updatedBy = agent;
+        this.writeBoard(projectName, board);
+
+        return { id, path: path.join(dir, fileName) };
+    }
+
+    // -- File Index --
+
+    readFileIndex(projectName) {
+        return readJson(this.fileIndexPath(projectName)) || { updatedAt: nowISO(), tree: {}, directories: {} };
+    }
+
+    writeFileIndex(projectName, index) {
+        index.updatedAt = nowISO();
+        writeJson(this.fileIndexPath(projectName), index);
+    }
+
+    // Auto-scan a directory and generate file-index tree
+    scanDirectory(rootDir, options = {}) {
+        const maxDepth = options.maxDepth || 5;
+        const excludes = options.excludes || ['node_modules', '.git', 'dist', 'out', '.next'];
+
+        function walk(dir, depth) {
+            if (depth > maxDepth) return {};
+            const result = {};
+            try {
+                const entries = fs.readdirSync(dir, { withFileTypes: true });
+                for (const entry of entries) {
+                    if (excludes.includes(entry.name)) continue;
+                    if (entry.name.startsWith('.') && entry.name !== '.env') continue;
+
+                    const fullPath = path.join(dir, entry.name);
+                    if (entry.isDirectory()) {
+                        const key = entry.name + '/';
+                        result[key] = {
+                            desc: '',
+                            children: walk(fullPath, depth + 1),
+                        };
+                    } else {
+                        const stat = fs.statSync(fullPath);
+                        result[entry.name] = {
+                            desc: '',
+                            created: stat.birthtime.toISOString().split('T')[0],
+                            modified: stat.mtime.toISOString().split('T')[0],
+                            status: 'active',
+                        };
+                    }
+                }
+            } catch (e) {
+                logError('scanDirectory', e);
+            }
+            return result;
+        }
+
+        return walk(rootDir, 0);
+    }
+}
+
+module.exports = { SoulEngine };

package/lib/utils.js

@@ -0,0 +1,97 @@
+// Soul MCP v4.0 — Shared utility functions (file I/O, time, security, logging)
+const fs = require('fs');
+const path = require('path');
+
+// -- Logging --
+
+function logError(context, err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    console.error(`[soul:${context}]`, msg);
+}
+
+// -- File I/O --
+
+function readFile(filePath) {
+    try {
+        return fs.readFileSync(filePath, 'utf8');
+    } catch (e) {
+        logError('readFile', e);
+        return null;
+    }
+}
+
+function readJson(filePath) {
+    const content = readFile(filePath);
+    if (!content) return null;
+    try {
+        return JSON.parse(content);
+    } catch (e) {
+        logError('readJson', e);
+        return null;
+    }
+}
+
+function writeJson(filePath, data) {
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf8');
+}
+
+function writeFile(filePath, content) {
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.writeFileSync(filePath, content, 'utf8');
+}
+
+// -- Time --
+
+function today() {
+    return new Date().toLocaleDateString('sv-SE', { timeZone: 'Asia/Seoul' });
+}
+
+function nowISO() {
+    const formatter = new Intl.DateTimeFormat('sv-SE', {
+        timeZone: 'Asia/Seoul',
+        year: 'numeric', month: '2-digit', day: '2-digit',
+        hour: '2-digit', minute: '2-digit', second: '2-digit',
+        hour12: false,
+    });
+    const parts = formatter.formatToParts(new Date());
+    const get = (type) => parts.find(p => p.type === type)?.value || '00';
+    return `${get('year')}-${get('month')}-${get('day')}T${get('hour')}:${get('minute')}:${get('second')}+09:00`;
+}
+
+// -- Security --
+
+function safePath(filePath, baseDir) {
+    const resolved = path.resolve(baseDir, filePath);
+    const normalizedBase = path.resolve(baseDir);
+    if (!resolved.startsWith(normalizedBase + path.sep) && resolved !== normalizedBase) {
+        logError('safePath', `Path traversal blocked: ${filePath}`);
+        return null;
+    }
+    return resolved;
+}
+
+// -- First-line comment validation --
+
+function validateFirstLineComment(filePath) {
+    try {
+        const content = fs.readFileSync(filePath, 'utf8');
+        const firstLine = content.split('\n')[0].trim();
+        const patterns = [
+            /^\/\/\s*.+/,     // JS/TS
+            /^#\s*.+/,        // Python/Shell/YAML
+            /^<!--\s*.+/,     // HTML/MD
+            /^\/\*\s*.+/,     // CSS/Java
+            /^\{.*"_desc"/,   // JSON with _desc field
+        ];
+        return patterns.some(p => p.test(firstLine));
+    } catch (e) {
+        logError('validateFirstLineComment', e);
+        return false;
+    }
+}
+
+module.exports = {
+    logError, readFile, readJson, writeJson, writeFile,
+    today, nowISO, safePath, validateFirstLineComment,
+};