mark-improving-agent 2.2.0

package/dist/core/memory/knowledge-graph.js

@@ -0,0 +1,151 @@
+/**
+ * Knowledge Graph - Agent K Architecture Integration
+ *
+ * Structured memory with temporal context and relationship tracking.
+ * Based on Agent K paper: 4-layer cognitive architecture with KG.
+ */
+import { randomUUID } from 'crypto';
+import { atomicWriteJSON, ensureDir, readJSON } from '../../storage/archive.js';
+const MAX_CONNECTIONS_PER_NODE = 20;
+function createDefaultNode(partial) {
+    const now = Date.now();
+    return {
+        ...partial,
+        id: `kg-${now}-${randomUUID().slice(0, 8)}`,
+        createdAt: now,
+        lastAccessed: now,
+    };
+}
+export function createKnowledgeGraph(dataDir) {
+    let nodes = new Map();
+    let edges = new Map();
+    let isDirty = false;
+    const nodesFile = `${dataDir}/knowledge-graph-nodes.json`;
+    const edgesFile = `${dataDir}/knowledge-graph-edges.json`;
+    async function persist() {
+        if (!isDirty)
+            return;
+        await atomicWriteJSON(nodesFile, Object.fromEntries(nodes));
+        await atomicWriteJSON(edgesFile, Object.fromEntries(edges));
+        isDirty = false;
+    }
+    async function boot() {
+        await ensureDir(dataDir);
+        const loadedNodes = await readJSON(nodesFile, {});
+        const loadedEdges = await readJSON(edgesFile, {});
+        nodes = new Map(Object.entries(loadedNodes || {}));
+        edges = new Map(Object.entries(loadedEdges || {}));
+    }
+    function addNode(partial) {
+        const node = createDefaultNode(partial);
+        nodes.set(node.id, node);
+        isDirty = true;
+        return node;
+    }
+    function addEdge(partial) {
+        // Validate both nodes exist
+        if (!nodes.has(partial.sourceId) || !nodes.has(partial.targetId)) {
+            return null;
+        }
+        // Limit connections per node
+        const sourceNode = nodes.get(partial.sourceId);
+        if (sourceNode.connections.length >= MAX_CONNECTIONS_PER_NODE) {
+            return null;
+        }
+        const edge = {
+            ...partial,
+            id: `kge-${Date.now()}-${randomUUID().slice(0, 8)}`,
+            createdAt: Date.now(),
+        };
+        edges.set(edge.id, edge);
+        // Update node connections
+        if (!sourceNode.connections.includes(partial.targetId)) {
+            sourceNode.connections.push(partial.targetId);
+        }
+        const targetNode = nodes.get(partial.targetId);
+        if (partial.bidirectional && !targetNode.connections.includes(partial.sourceId)) {
+            targetNode.connections.push(partial.sourceId);
+        }
+        isDirty = true;
+        return edge;
+    }
+    function getNode(id) {
+        const node = nodes.get(id) ?? null;
+        if (node) {
+            node.lastAccessed = Date.now();
+            isDirty = true;
+        }
+        return node;
+    }
+    function getConnectedNodes(nodeId, relation) {
+        const node = nodes.get(nodeId);
+        if (!node)
+            return [];
+        const connected = [];
+        for (const edge of edges.values()) {
+            if (edge.sourceId === nodeId || (edge.bidirectional && edge.targetId === nodeId)) {
+                if (relation && edge.relation !== relation)
+                    continue;
+                const connectedId = edge.sourceId === nodeId ? edge.targetId : edge.sourceId;
+                const connectedNode = nodes.get(connectedId);
+                if (connectedNode) {
+                    connectedNode.lastAccessed = Date.now();
+                    connected.push(connectedNode);
+                }
+            }
+        }
+        return connected;
+    }
+    function search(query) {
+        const lowerQuery = query.toLowerCase();
+        const results = [];
+        for (const node of nodes.values()) {
+            if (node.name.toLowerCase().includes(lowerQuery) ||
+                node.description.toLowerCase().includes(lowerQuery)) {
+                results.push(node);
+            }
+        }
+        return results.sort((a, b) => b.importance - a.importance);
+    }
+    /**
+     * Reflect on a node - increment reflection count and update status
+     * Based on Agent K's reflection mechanism
+     */
+    function reflect(nodeId) {
+        const node = nodes.get(nodeId);
+        if (!node)
+            return null;
+        node.reflectionCount = (node.reflectionCount ?? 0) + 1;
+        node.lastAccessed = Date.now();
+        isDirty = true;
+        return node;
+    }
+    function getStats() {
+        let totalConnections = 0;
+        let mostConnected = [];
+        for (const node of nodes.values()) {
+            totalConnections += node.connections.length;
+            if (node.connections.length > 0) {
+                mostConnected.push({ id: node.id, count: node.connections.length });
+            }
+        }
+        mostConnected.sort((a, b) => b.count - a.count);
+        return {
+            nodes: nodes.size,
+            edges: edges.size,
+            avgConnections: nodes.size > 0 ? totalConnections / nodes.size : 0,
+            mostConnected: mostConnected.slice(0, 5).map(m => m.id),
+        };
+    }
+    return {
+        addNode,
+        addEdge,
+        getNode,
+        getConnectedNodes,
+        search,
+        reflect,
+        getStats,
+        persist,
+        boot,
+    };
+}

package/dist/core/memory/spaced-repetition.js

@@ -0,0 +1,113 @@
+// SM-2 Spaced Repetition Algorithm
+// Based on SuperMemo 2 algorithm for memory reinforcement
+import { randomUUID } from 'crypto';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { dirname } from 'path';
+export function createSpacedRepetition(dataDir) {
+    const stateFile = `${dataDir}/sr-state.json`;
+    let state = { entries: {} };
+    function loadState() {
+        try {
+            if (existsSync(stateFile)) {
+                const raw = readFileSync(stateFile, 'utf-8');
+                const parsed = JSON.parse(raw);
+                state = parsed;
+            }
+        }
+        catch {
+            // Use defaults
+        }
+    }
+    function persistState() {
+        try {
+            const dir = dirname(stateFile);
+            if (!existsSync(dir)) {
+                mkdirSync(dir, { recursive: true });
+            }
+            const tmp = stateFile + '.tmp.' + randomUUID().slice(0, 8);
+            writeFileSync(tmp, JSON.stringify(state, null, 2), 'utf-8');
+            require('fs').renameSync(tmp, stateFile);
+        }
+        catch {
+            // Persistence failure is non-fatal
+        }
+    }
+    // Load state on creation
+    loadState();
+    function recordReview(id, quality) {
+        const now = Date.now();
+        let entry = state.entries[id];
+        if (!entry) {
+            entry = {
+                id,
+                easeFactor: 2.5,
+                interval: 0,
+                repetitions: 0,
+                nextReviewAt: now,
+                lastReviewedAt: now,
+            };
+        }
+        // SM-2 algorithm
+        if (quality >= 3) {
+            // Successful recall
+            if (entry.repetitions === 0) {
+                entry.interval = 1;
+            }
+            else if (entry.repetitions === 1) {
+                entry.interval = 6;
+            }
+            else {
+                entry.interval = Math.round(entry.interval * entry.easeFactor);
+            }
+            entry.repetitions++;
+        }
+        else {
+            // Failed recall - reset
+            entry.interval = 1;
+            entry.repetitions = 0;
+        }
+        // Update ease factor: EF' = EF + (0.1 - (5-q) * (0.08 + (5-q) * 0.02))
+        const q = quality;
+        entry.easeFactor = Math.max(1.3, entry.easeFactor + (0.1 - (5 - q) * (0.08 + (5 - q) * 0.02)));
+        // Set next review time
+        entry.nextReviewAt = now + entry.interval * 24 * 60 * 60 * 1000;
+        entry.lastReviewedAt = now;
+        state.entries[id] = entry;
+        persistState();
+    }
+    function getNextReviewDate(id) {
+        const entry = state.entries[id];
+        return entry ? entry.nextReviewAt : null;
+    }
+    function getDueEntries(entries) {
+        const now = Date.now();
+        return entries.filter(entry => {
+            const srEntry = state.entries[entry.id];
+            return !srEntry || srEntry.nextReviewAt <= now;
+        });
+    }
+    function getSpacedRepetitionStats() {
+        const now = Date.now();
+        let dueCount = 0;
+        let masteredCount = 0;
+        let learningCount = 0;
+        for (const entry of Object.values(state.entries)) {
+            if (entry.nextReviewAt <= now) {
+                dueCount++;
+            }
+            if (entry.repetitions >= 5 && entry.easeFactor >= 2.5) {
+                masteredCount++;
+            }
+            else if (entry.repetitions > 0) {
+                learningCount++;
+            }
+        }
+        return { dueCount, masteredCount, learningCount };
+    }
+    return {
+        recordReview,
+        getNextReviewDate,
+        getDueEntries,
+        getSpacedRepetitionStats,
+    };
+}

package/dist/core/memory/store.js

@@ -0,0 +1,404 @@
+import { randomUUID } from 'crypto';
+import { atomicWriteJSON, ensureDir, readJSON } from '../../storage/archive.js';
+import { cosineSimilarity } from './embedder.js';
+import { events } from '../../event/bus.js';
+import { createDreamEngine } from './dream-consolidation.js';
+import { createSpacedRepetition } from './spaced-repetition.js';
+const STORE_VERSION = 1;
+const SAVE_DEBOUNCE_MS = 500;
+/**
+ * Source weight constants for importance scoring.
+ */
+const SOURCE_WEIGHTS = {
+    user_correction: 1.0,
+    decision_verified: 0.85,
+    self: 0.7,
+    system: 0.5,
+};
+function computeImportance(baseImportance, source, timestamp) {
+    const sourceWeight = source ? SOURCE_WEIGHTS[source] ?? 0.5 : 0.5;
+    const ageMs = Date.now() - timestamp;
+    const ageDays = ageMs / (1000 * 60 * 60 * 24);
+    const recencyBonus = Math.max(0, 0.1 - ageDays * 0.01);
+    return Math.min(1.0, baseImportance * sourceWeight + recencyBonus);
+}
+export function createMemoryStore(options) {
+    const dataDir = options.dataDir;
+    const embedder = options.embedder;
+    const learnedTtlDays = options.learnedTtlDays ?? 30;
+    const maxEphemeral = options.maxEphemeral ?? 1000;
+    // Three separate in-memory Maps
+    const coreMap = new Map();
+    const learnedMap = new Map();
+    const ephemeralMap = new Map();
+    // Dream engine and spaced repetition
+    const dreamEngine = createDreamEngine(dataDir);
+    const spacedRepetition = createSpacedRepetition(dataDir);
+    // File paths
+    const coreFile = `${dataDir}/core.json`;
+    const learnedFile = `${dataDir}/learned.json`;
+    const ephemeralFile = `${dataDir}/ephemeral.json`;
+    // Dirty flags and debounce timers
+    let coreDirty = false;
+    let learnedDirty = false;
+    let ephemeralDirty = false;
+    let saveTimer = null;
+    function getTierMap(tier) {
+        switch (tier) {
+            case 'core': return coreMap;
+            case 'learned': return learnedMap;
+            case 'ephemeral': return ephemeralMap;
+        }
+    }
+    function getTierFile(tier) {
+        switch (tier) {
+            case 'core': return coreFile;
+            case 'learned': return learnedFile;
+            case 'ephemeral': return ephemeralFile;
+        }
+    }
+    function markDirty(tier) {
+        switch (tier) {
+            case 'core':
+                coreDirty = true;
+                break;
+            case 'learned':
+                learnedDirty = true;
+                break;
+            case 'ephemeral':
+                ephemeralDirty = true;
+                break;
+        }
+        scheduleSave();
+    }
+    async function saveTier(tier) {
+        const map = getTierMap(tier);
+        const file = getTierFile(tier);
+        const data = { entries: Object.fromEntries(map) };
+        await atomicWriteJSON(file, data);
+        switch (tier) {
+            case 'core':
+                coreDirty = false;
+                break;
+            case 'learned':
+                learnedDirty = false;
+                break;
+            case 'ephemeral':
+                ephemeralDirty = false;
+                break;
+        }
+    }
+    function scheduleSave() {
+        if (saveTimer !== null) {
+            clearTimeout(saveTimer);
+        }
+        saveTimer = setTimeout(async () => {
+            try {
+                await Promise.all([
+                    coreDirty ? saveTier('core') : Promise.resolve(),
+                    learnedDirty ? saveTier('learned') : Promise.resolve(),
+                    ephemeralDirty ? saveTier('ephemeral') : Promise.resolve(),
+                ]);
+            }
+            catch (err) {
+                console.error('Memory store auto-save failed:', err);
+            }
+            saveTimer = null;
+        }, SAVE_DEBOUNCE_MS);
+    }
+    async function saveAllIfDirty() {
+        await Promise.all([
+            coreDirty ? saveTier('core') : Promise.resolve(),
+            learnedDirty ? saveTier('learned') : Promise.resolve(),
+            ephemeralDirty ? saveTier('ephemeral') : Promise.resolve(),
+        ]);
+    }
+    async function evictOldestEphemeral() {
+        const entries = Array.from(ephemeralMap.values());
+        entries.sort((a, b) => a.timestamp - b.timestamp);
+        const toRemove = Math.ceil(ephemeralMap.size * 0.1); // Remove 10%
+        for (let i = 0; i < toRemove && i < entries.length; i++) {
+            ephemeralMap.delete(entries[i].id);
+        }
+        if (toRemove > 0) {
+            markDirty('ephemeral');
+        }
+    }
+    // Memory store implementation with Agent K enhancements
+    const store = (content, importance, tags = [], tier = 'learned', source = 'system', options) => {
+        const id = randomUUID();
+        const timestamp = Date.now();
+        const embedding = embedder.embed(content);
+        // Determine effective tier based on importance
+        let effectiveTier = tier;
+        if (tier === 'learned' && importance > 0.85) {
+            effectiveTier = 'core';
+        }
+        const entry = {
+            id,
+            tier: effectiveTier,
+            content,
+            importance,
+            tags,
+            timestamp,
+            accessCount: 0,
+            lastAccessed: timestamp,
+            ttl: effectiveTier === 'learned' ? learnedTtlDays * 24 * 60 * 60 * 1000 : undefined,
+            source,
+            embedding,
+            // Agent K enhancements
+            temporalContext: options?.temporalContext ?? {
+                isOngoing: true,
+            },
+            reflectionCount: 0,
+            verificationStatus: 'unverified',
+        };
+        getTierMap(effectiveTier).set(id, entry);
+        markDirty(effectiveTier);
+        // Record successful store as spaced repetition review (quality=4)
+        spacedRepetition.recordReview(id, 4);
+        // Enforce ephemeral limit
+        if (ephemeralMap.size > maxEphemeral) {
+            evictOldestEphemeral();
+        }
+        events.emit('memory:store', entry);
+        return entry;
+    };
+    const recall = async (id) => {
+        let entry = coreMap.get(id) ?? learnedMap.get(id) ?? ephemeralMap.get(id);
+        if (entry) {
+            entry = { ...entry, accessCount: entry.accessCount + 1, lastAccessed: Date.now() };
+            getTierMap(entry.tier).set(id, entry);
+            events.emit('memory:recall', { id: entry.id, found: true });
+        }
+        return entry ?? null;
+    };
+    const forget = async (id) => {
+        for (const [tid, map] of [['core', coreMap], ['learned', learnedMap], ['ephemeral', ephemeralMap]]) {
+            if (map.has(id)) {
+                map.delete(id);
+                markDirty(tid);
+                return true;
+            }
+        }
+        return false;
+    };
+    const search = (query, limit = 10) => {
+        const queryEmbedding = embedder.embed(query);
+        const results = [];
+        const allEntries = [
+            ...Array.from(coreMap.values()),
+            ...Array.from(learnedMap.values()),
+            ...Array.from(ephemeralMap.values()),
+        ];
+        for (const entry of allEntries) {
+            if (!entry.embedding)
+                continue;
+            const score = cosineSimilarity(queryEmbedding, entry.embedding);
+            if (score > 0.5) {
+                results.push({
+                    entry,
+                    score,
+                    reason: 'semantic',
+                });
+            }
+        }
+        // Sort by score descending
+        results.sort((a, b) => b.score - a.score);
+        // Reinforce memories on successful recall (spaced repetition)
+        for (const result of results) {
+            spacedRepetition.recordReview(result.entry.id, 4);
+        }
+        return results.slice(0, limit);
+    };
+    const searchByTags = async (tags, limit = 20) => {
+        const lowerTags = tags.map(t => t.toLowerCase());
+        const results = [];
+        const allEntries = [
+            ...Array.from(coreMap.values()),
+            ...Array.from(learnedMap.values()),
+            ...Array.from(ephemeralMap.values()),
+        ];
+        for (const entry of allEntries) {
+            if (entry.tags.some(tag => lowerTags.includes(tag.toLowerCase()))) {
+                results.push(entry);
+            }
+        }
+        // Sort by importance descending
+        results.sort((a, b) => b.importance - a.importance);
+        return results.slice(0, limit);
+    };
+    const recallRecent = async (limit = 20) => {
+        const results = [
+            ...Array.from(coreMap.values()),
+            ...Array.from(learnedMap.values()),
+            ...Array.from(ephemeralMap.values()),
+        ];
+        results.sort((a, b) => b.timestamp - a.timestamp);
+        return results.slice(0, limit);
+    };
+    const recallHighValue = async (minImportance = 0.8) => {
+        const results = [
+            ...Array.from(coreMap.values()),
+            ...Array.from(learnedMap.values()),
+            ...Array.from(ephemeralMap.values()),
+        ];
+        return results
+            .filter(e => e.importance >= minImportance)
+            .sort((a, b) => b.importance - a.importance);
+    };
+    const distill = async () => {
+        let promoted = 0;
+        const now = Date.now();
+        for (const [id, entry] of learnedMap) {
+            // Promote learned entries with high importance and age
+            const ageMs = now - entry.timestamp;
+            const ageDays = ageMs / (1000 * 60 * 60 * 24);
+            if (entry.importance > 0.85 || ageDays > 60) {
+                learnedMap.delete(id);
+                const promotedEntry = {
+                    ...entry,
+                    tier: 'core',
+                    importance: Math.min(1, entry.importance + 0.05),
+                };
+                coreMap.set(id, promotedEntry);
+                markDirty('learned');
+                markDirty('core');
+                promoted++;
+            }
+        }
+        return promoted;
+    };
+    const prune = async () => {
+        let pruned = 0;
+        const now = Date.now();
+        for (const [id, entry] of learnedMap) {
+            if (entry.ttl) {
+                const ageMs = now - entry.timestamp;
+                if (ageMs > entry.ttl) {
+                    learnedMap.delete(id);
+                    markDirty('learned');
+                    pruned++;
+                }
+            }
+        }
+        return pruned;
+    };
+    const getStats = async () => {
+        const allEntries = [
+            ...Array.from(coreMap.values()),
+            ...Array.from(learnedMap.values()),
+            ...Array.from(ephemeralMap.values()),
+        ];
+        const avgImportance = allEntries.length > 0
+            ? allEntries.reduce((sum, e) => sum + e.importance, 0) / allEntries.length
+            : 0;
+        return {
+            core: coreMap.size,
+            learned: learnedMap.size,
+            ephemeral: ephemeralMap.size,
+            avgImportance,
+            archived: 0,
+        };
+    };
+    const boot = async () => {
+        await ensureDir(dataDir);
+        const loadTier = async (file) => {
+            const map = new Map();
+            const data = await readJSON(file, { entries: {} });
+            if (data && data.entries) {
+                for (const [id, entry] of Object.entries(data.entries)) {
+                    map.set(id, entry);
+                }
+            }
+            return map;
+        };
+        try {
+            const loadedCore = await loadTier(coreFile);
+            if (loadedCore.size > 0)
+                coreMap.clear();
+            loadedCore.forEach((v, k) => coreMap.set(k, v));
+        }
+        catch (err) {
+            console.error(`Failed to load core tier from ${coreFile}:`, err);
+        }
+        try {
+            const loadedLearned = await loadTier(learnedFile);
+            if (loadedLearned.size > 0)
+                learnedMap.clear();
+            loadedLearned.forEach((v, k) => learnedMap.set(k, v));
+        }
+        catch (err) {
+            console.error(`Failed to load learned tier from ${learnedFile}:`, err);
+        }
+        try {
+            const loadedEphemeral = await loadTier(ephemeralFile);
+            if (loadedEphemeral.size > 0)
+                ephemeralMap.clear();
+            loadedEphemeral.forEach((v, k) => ephemeralMap.set(k, v));
+        }
+        catch (err) {
+            console.error(`Failed to load ephemeral tier from ${ephemeralFile}:`, err);
+        }
+    };
+    const shutdown = async () => {
+        if (saveTimer) {
+            clearTimeout(saveTimer);
+            saveTimer = null;
+        }
+        await saveAllIfDirty();
+    };
+    const exportAll = () => {
+        const data = {
+            version: STORE_VERSION,
+            core: { entries: Object.fromEntries(coreMap) },
+            learned: { entries: Object.fromEntries(learnedMap) },
+            ephemeral: { entries: Object.fromEntries(ephemeralMap) },
+        };
+        return JSON.stringify(data, null, 2);
+    };
+    // Dream consolidation
+    async function dream() {
+        const allEntries = [
+            ...Array.from(coreMap.values()),
+            ...Array.from(learnedMap.values()),
+            ...Array.from(ephemeralMap.values()),
+        ];
+        return dreamEngine.dream(allEntries);
+    }
+    // Spaced repetition
+    function getDueMemories() {
+        const allEntries = [
+            ...Array.from(coreMap.values()),
+            ...Array.from(learnedMap.values()),
+            ...Array.from(ephemeralMap.values()),
+        ];
+        return spacedRepetition.getDueEntries(allEntries);
+    }
+    function recordReview(id, quality) {
+        spacedRepetition.recordReview(id, quality);
+    }
+    // Memory health
+    function getMemoryHealth() {
+        return dreamEngine.getMemoryHealth();
+    }
+    return {
+        store,
+        recall,
+        forget,
+        search,
+        searchByTags,
+        recallRecent,
+        recallHighValue,
+        distill,
+        prune,
+        getStats,
+        boot,
+        shutdown,
+        exportAll,
+        dream,
+        getDueMemories,
+        recordReview,
+        getMemoryHealth,
+    };
+}

package/dist/core/memory/types.js

@@ -0,0 +1 @@
+export {};