mark-improving-agent 2.2.3 → 2.2.4

package/dist/core/identity/identity-continuity.js

@@ -0,0 +1,301 @@
+/**
+ * Identity Continuity Verifier
+ *
+ * Monitors and maintains identity coherence across sessions.
+ * Based on HeartFlow's identity architecture - ensuring the AI
+ * remains "the same person" over time and across contexts.
+ *
+ * Key mechanisms:
+ * - Core directive alignment checking
+ * - Belief drift detection
+ * - Value consistency verification
+ * - Identity checkpointing
+ *
+ * @module core/identity
+ * @fileoverview Identity continuity and coherence verification
+ */
+import { randomUUID } from 'crypto';
+import { createLogger } from '../../utils/logger.js';
+const logger = createLogger('[IdentityContinuity]');
+const DEFAULT_CONFIG = {
+    maxBeliefDrift: 0.2,
+    checkpointIntervalDays: 1,
+    healthyThreshold: 0.85,
+    maxPersonalityShift: 15,
+};
+function simpleHash(input) {
+    let hash = 0;
+    const str = JSON.stringify(input);
+    for (let i = 0; i < str.length; i++) {
+        const char = str.charCodeAt(i);
+        hash = ((hash << 5) - hash) + char;
+        hash = hash & hash;
+    }
+    return Math.abs(hash).toString(16).padStart(8, '0');
+}
+function cosineSimilarity(a, b) {
+    if (a.length !== b.length)
+        return 0;
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dotProduct += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+    return denominator === 0 ? 0 : dotProduct / denominator;
+}
+function personalityToVector(personality) {
+    const keys = Object.keys(personality).sort();
+    return keys.map((k) => personality[k] ?? 0);
+}
+function extractConfidenceFromBelief(beliefStr) {
+    const match = beliefStr.match(/conf[:\s]*([0-9.]+)/i);
+    return match ? parseFloat(match[1]) : 0.7;
+}
+export function createIdentityContinuityVerifier(config = {}) {
+    const cfg = { ...DEFAULT_CONFIG, ...config };
+    let checkpoints = [];
+    let violations = [];
+    let totalVerifications = 0;
+    let avgContinuityScore = 1.0;
+    function verify(currentBeliefs, currentValues, currentPersonality, currentDirectives) {
+        totalVerifications++;
+        const now = Date.now();
+        const newViolations = [];
+        const lastCheckpoint = checkpoints.length > 0
+            ? checkpoints[checkpoints.length - 1]
+            : null;
+        if (!lastCheckpoint) {
+            avgContinuityScore = 1.0;
+            return {
+                checkpoints,
+                violations: [],
+                overallContinuityScore: 1.0,
+                riskLevel: 'healthy',
+                recommendations: ['First verification - no baseline to compare. A checkpoint will establish your identity baseline.'],
+                lastVerified: now,
+            };
+        }
+        // 1. Check belief drift
+        const lastBeliefs = lastCheckpoint.beliefs;
+        for (const currentBelief of currentBeliefs) {
+            const matchingBelief = lastBeliefs.find((b) => b.toLowerCase().trim() === currentBelief.content.toLowerCase().trim());
+            if (matchingBelief) {
+                const prevConfidence = extractConfidenceFromBelief(matchingBelief);
+                const drift = Math.abs(prevConfidence - currentBelief.confidence);
+                if (drift > cfg.maxBeliefDrift) {
+                    newViolations.push({
+                        id: `violation-${randomUUID().slice(0, 8)}`,
+                        type: 'belief_drift',
+                        severity: drift > 0.5 ? 'high' : 'medium',
+                        description: `Belief "${currentBelief.content.slice(0, 50)}..." confidence shifted by ${(drift * 100).toFixed(1)}%`,
+                        previousValue: `confidence: ${prevConfidence.toFixed(2)}`,
+                        currentValue: `confidence: ${currentBelief.confidence.toFixed(2)}`,
+                        timestamp: now,
+                        resolved: false,
+                    });
+                }
+            }
+        }
+        // 2. Check value consistency
+        const prevValuesSet = new Set(lastCheckpoint.values.map((v) => v.toLowerCase()));
+        for (const currentValue of currentValues) {
+            if (!prevValuesSet.has(currentValue.toLowerCase())) {
+                const removedValue = lastCheckpoint.values.find((v) => v.toLowerCase() !== currentValue.toLowerCase());
+                if (removedValue) {
+                    newViolations.push({
+                        id: `violation-${randomUUID().slice(0, 8)}`,
+                        type: 'value_mismatch',
+                        severity: 'medium',
+                        description: `Value set changed - previously had "${removedValue}"`,
+                        previousValue: removedValue,
+                        currentValue: currentValue,
+                        timestamp: now,
+                        resolved: false,
+                    });
+                }
+            }
+        }
+        // 3. Check core directive alignment
+        const prevDirectivesSet = new Set(lastCheckpoint.directives.map((d) => d.toLowerCase()));
+        for (const currentDirective of currentDirectives) {
+            if (!prevDirectivesSet.has(currentDirective.toLowerCase())) {
+                newViolations.push({
+                    id: `violation-${randomUUID().slice(0, 8)}`,
+                    type: 'directive_conflict',
+                    severity: 'critical',
+                    description: `Core directive deviation detected`,
+                    previousValue: lastCheckpoint.directives.join(', '),
+                    currentValue: currentDirectives.join(', '),
+                    timestamp: now,
+                    resolved: false,
+                });
+            }
+        }
+        // 4. Check personality shift
+        const prevPersonalityVector = personalityToVector(lastCheckpoint.personalitySnapshot);
+        const currentPersonalityVector = personalityToVector(currentPersonality);
+        if (prevPersonalityVector.length === currentPersonalityVector.length) {
+            const similarity = cosineSimilarity(prevPersonalityVector, currentPersonalityVector);
+            const shift = 1 - similarity;
+            if (shift > cfg.maxPersonalityShift / 100) {
+                newViolations.push({
+                    id: `violation-${randomUUID().slice(0, 8)}`,
+                    type: 'personality_shift',
+                    severity: shift > 0.3 ? 'high' : 'medium',
+                    description: `Personality profile shifted by ${(shift * 100).toFixed(1)}%`,
+                    previousValue: JSON.stringify(lastCheckpoint.personalitySnapshot),
+                    currentValue: JSON.stringify(currentPersonality),
+                    timestamp: now,
+                    resolved: false,
+                });
+            }
+        }
+        // 5. Self-model hash check
+        const currentHash = hashSelfModel(currentBeliefs, currentValues, currentDirectives);
+        if (currentHash !== lastCheckpoint.selfModelHash) {
+            const hasOtherViolations = newViolations.length > 0;
+            if (!hasOtherViolations) {
+                newViolations.push({
+                    id: `violation-${randomUUID().slice(0, 8)}`,
+                    type: 'self_model_inconsistency',
+                    severity: 'low',
+                    description: 'Self-model hash changed without specific violation detected',
+                    previousValue: lastCheckpoint.selfModelHash,
+                    currentValue: currentHash,
+                    timestamp: now,
+                    resolved: false,
+                });
+            }
+        }
+        // Calculate overall continuity score
+        let continuityScore = 1.0;
+        for (const v of newViolations) {
+            switch (v.severity) {
+                case 'critical':
+                    continuityScore -= 0.4;
+                    break;
+                case 'high':
+                    continuityScore -= 0.2;
+                    break;
+                case 'medium':
+                    continuityScore -= 0.1;
+                    break;
+                case 'low':
+                    continuityScore -= 0.05;
+                    break;
+            }
+        }
+        continuityScore = Math.max(0, continuityScore);
+        avgContinuityScore = (avgContinuityScore * (totalVerifications - 1) + continuityScore) / totalVerifications;
+        let riskLevel = 'healthy';
+        if (continuityScore < 0.5)
+            riskLevel = 'critical';
+        else if (continuityScore < cfg.healthyThreshold - 0.15)
+            riskLevel = 'concerning';
+        else if (continuityScore < cfg.healthyThreshold)
+            riskLevel = 'watch';
+        const recommendations = [];
+        if (riskLevel !== 'healthy') {
+            recommendations.push(`Identity continuity score is ${(continuityScore * 100).toFixed(1)}% - ${riskLevel} status`);
+            const unresolved = newViolations.filter((v) => !v.resolved);
+            if (unresolved.length > 0) {
+                recommendations.push(`${unresolved.length} unresolved violation(s) require attention`);
+            }
+        }
+        const criticalViolations = newViolations.filter((v) => v.severity === 'critical');
+        if (criticalViolations.length > 0) {
+            recommendations.push('Critical identity violations detected - review core directives immediately');
+        }
+        if (checkpoints.length === 0) {
+            recommendations.push('No checkpoints on record - consider creating a baseline checkpoint');
+        }
+        const timeSinceCheckpoint = getTimeSinceLastCheckpointMs();
+        if (timeSinceCheckpoint > cfg.checkpointIntervalDays * 24 * 60 * 60 * 1000) {
+            recommendations.push(`Last checkpoint was ${Math.floor(timeSinceCheckpoint / (24 * 60 * 60 * 1000))} day(s) ago - consider updating`);
+        }
+        if (recommendations.length === 0) {
+            recommendations.push('Identity continuity is healthy');
+        }
+        // Add new violations to state
+        violations.push(...newViolations);
+        logger.info(`Continuity verification complete: score=${continuityScore.toFixed(3)}, violations=${newViolations.length}`);
+        return {
+            checkpoints,
+            violations: [...violations],
+            overallContinuityScore: continuityScore,
+            riskLevel,
+            recommendations,
+            lastVerified: now,
+        };
+    }
+    function hashSelfModel(beliefs, values, directives) {
+        const beliefStrings = beliefs.map((b) => `${b.content}::${b.confidence}`);
+        const combined = [...beliefStrings, ...values, ...directives].sort().join('|');
+        return simpleHash(combined);
+    }
+    function checkpoint(beliefs, values, directives, personality, selfModelHash, notes) {
+        const newCheckpoint = {
+            id: `checkpoint-${randomUUID().slice(0, 8)}`,
+            timestamp: Date.now(),
+            beliefs: beliefs.map((b) => `${b.content} (conf: ${b.confidence.toFixed(2)})`),
+            values: [...values],
+            directives: [...directives],
+            personalitySnapshot: { ...personality },
+            selfModelHash,
+            notes,
+        };
+        checkpoints.push(newCheckpoint);
+        if (checkpoints.length > 50) {
+            checkpoints = checkpoints.slice(-50);
+        }
+        logger.info(`Identity checkpoint created: ${newCheckpoint.id}`);
+        return newCheckpoint;
+    }
+    function getCheckpoints(limit) {
+        const sorted = [...checkpoints].sort((a, b) => b.timestamp - a.timestamp);
+        return limit ? sorted.slice(0, limit) : sorted;
+    }
+    function getViolations() {
+        return violations.filter((v) => !v.resolved);
+    }
+    function resolveViolation(violationId) {
+        const violation = violations.find((v) => v.id === violationId);
+        if (violation) {
+            violation.resolved = true;
+            logger.info(`Violation resolved: ${violationId}`);
+        }
+    }
+    function getTimeSinceLastCheckpointMs() {
+        if (checkpoints.length === 0)
+            return Infinity;
+        const last = checkpoints[checkpoints.length - 1];
+        return Date.now() - last.timestamp;
+    }
+    function forceCheckpoint() {
+        logger.info('Force checkpoint requested - call checkpoint() with current identity state');
+    }
+    function getStats() {
+        const resolvedCount = violations.filter((v) => v.resolved).length;
+        return {
+            totalCheckpoints: checkpoints.length,
+            totalViolations: violations.length,
+            resolvedViolations: resolvedCount,
+            avgContinuityScore,
+        };
+    }
+    return {
+        verify,
+        checkpoint,
+        getCheckpoints,
+        getViolations,
+        resolveViolation,
+        hashSelfModel,
+        getTimeSinceLastCheckpoint: () => getTimeSinceLastCheckpointMs(),
+        forceCheckpoint,
+        getStats,
+    };
+}

package/dist/core/identity/index.js

@@ -1,3 +1,4 @@
 export * from './types.js';
 export * from './self-model.js';
 export { createReflexion } from './reflexion.js';
+export { createIdentityContinuityVerifier } from './identity-continuity.js';

package/dist/core/memory/context-fragmentation.js

@@ -0,0 +1,308 @@
+/**
+ * Context Fragmentation Detector
+ *
+ * Detects and prevents memory fragmentation - the tendency for
+ * episodic memories to become disconnected from core identity
+ * and semantic knowledge over time.
+ *
+ * Based on Memex's fragment-based journaling concept and
+ * hippocampal consolidation theory.
+ *
+ * Key mechanisms:
+ * - Fragment detection: find isolated memory clusters
+ * - Connection scoring: measure semantic coherence
+ * - Consolidation triggers: suggest when memories need linking
+ * - Fragment repair: create bridges between isolated nodes
+ *
+ * @module core/memory
+ * @fileoverview Context fragmentation detection and repair
+ */
+import { randomUUID } from 'crypto';
+import { createLogger } from '../../utils/logger.js';
+const logger = createLogger('[ContextFragmentation]');
+const DEFAULT_CONFIG = {
+    minClusterSize: 2,
+    isolationThresholdDays: 7,
+    minHealthyCoherence: 0.4,
+    maxHealthyFragmentSize: 20,
+};
+/**
+ * Extract semantic theme from a set of memory contents
+ */
+function extractTheme(memoryIds, getNode) {
+    const contents = [];
+    for (const id of memoryIds) {
+        const node = getNode(id);
+        if (node) {
+            contents.push(node.content);
+            contents.push(...node.tags);
+        }
+    }
+    if (contents.length === 0)
+        return 'unknown';
+    // Simple keyword extraction - find most common meaningful words
+    const words = {};
+    const stopWords = new Set(['the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need', 'dare', 'ought', 'used', 'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'between', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 'just', 'but', 'and', 'or', 'if', 'because', 'until', 'while', 'although', '这', '的', '是', '在', '了', '和', '与', '或', '我', '你', '他', '她', '它', '我们', '你们', '他们']);
+    for (const content of contents) {
+        const tokens = content.toLowerCase().split(/\s+/);
+        for (const token of tokens) {
+            const cleaned = token.replace(/[^a-zA-Z\u4e00-\u9fff]/g, '');
+            if (cleaned.length > 2 && !stopWords.has(cleaned)) {
+                words[cleaned] = (words[cleaned] || 0) + 1;
+            }
+        }
+    }
+    // Sort by frequency
+    const sorted = Object.entries(words).sort((a, b) => b[1] - a[1]);
+    if (sorted.length === 0)
+        return 'unknown';
+    // Return top 2-3 keywords as theme
+    const topWords = sorted.slice(0, 3).map(([w]) => w);
+    return topWords.join(' + ');
+}
+/**
+ * Build adjacency list from memory connections
+ */
+function buildAdjacencyList(memoryIds, getNode) {
+    const adj = new Map();
+    for (const id of memoryIds) {
+        if (!adj.has(id)) {
+            adj.set(id, new Set());
+        }
+        const node = getNode(id);
+        if (node) {
+            for (const connId of node.connections) {
+                adj.get(id).add(connId);
+                if (!adj.has(connId)) {
+                    adj.set(connId, new Set());
+                }
+                adj.get(connId).add(id);
+            }
+        }
+    }
+    return adj;
+}
+/**
+ * Find connected components (fragments) using BFS
+ */
+function findConnectedComponents(memoryIds, adj) {
+    const visited = new Set();
+    const components = [];
+    function bfs(startId) {
+        const component = [];
+        const queue = [startId];
+        while (queue.length > 0) {
+            const current = queue.shift();
+            if (visited.has(current))
+                continue;
+            visited.add(current);
+            component.push(current);
+            const neighbors = adj.get(current) || new Set();
+            for (const neighbor of neighbors) {
+                if (!visited.has(neighbor)) {
+                    queue.push(neighbor);
+                }
+            }
+        }
+        return component;
+    }
+    for (const id of memoryIds) {
+        if (!visited.has(id)) {
+            const component = bfs(id);
+            if (component.length > 0) {
+                components.push(component);
+            }
+        }
+    }
+    return components;
+}
+export function createContextFragmentationEngine(config = {}) {
+    const cfg = { ...DEFAULT_CONFIG, ...config };
+    let totalAnalyzed = 0;
+    let avgFragmentationScore = 0;
+    let lastAnalysis = null;
+    function analyze(memoryIds, getNode) {
+        totalAnalyzed++;
+        lastAnalysis = Date.now();
+        if (memoryIds.length === 0) {
+            return {
+                fragments: [],
+                isolatedFragments: [],
+                orphanedMemories: [],
+                overallFragmentationScore: 0,
+                recommendations: ['No memories to analyze'],
+                timestamp: lastAnalysis,
+            };
+        }
+        const adj = buildAdjacencyList(memoryIds, getNode);
+        const components = findConnectedComponents(memoryIds, adj);
+        // Identify core-connected memories (tier = 'core')
+        const coreIds = new Set();
+        for (const id of memoryIds) {
+            const node = getNode(id);
+            if (node && node.tier === 'core') {
+                coreIds.add(id);
+            }
+        }
+        // Build fragments from components
+        const fragments = components.map((componentIds) => {
+            const coherence = calculateCoherence(componentIds, getNode);
+            // Find last core connection
+            let lastCoreConnection = 0;
+            for (const id of componentIds) {
+                const node = getNode(id);
+                if (node) {
+                    // Check if any connected node is core
+                    for (const connId of node.connections) {
+                        if (coreIds.has(connId)) {
+                            lastCoreConnection = Math.max(lastCoreConnection, node.timestamp);
+                        }
+                    }
+                }
+            }
+            return {
+                id: `fragment-${randomUUID().slice(0, 8)}`,
+                memoryIds: componentIds,
+                theme: extractTheme(componentIds, getNode),
+                internalCoherence: coherence,
+                lastCoreConnection,
+                size: componentIds.length,
+            };
+        });
+        // Identify isolated fragments
+        const now = Date.now();
+        const isolationThresholdMs = cfg.isolationThresholdDays * 24 * 60 * 60 * 1000;
+        const isolatedFragments = fragments.filter((f) => {
+            // No core connections at all
+            if (f.lastCoreConnection === 0)
+                return true;
+            // Hasn't connected to core in threshold time
+            return now - f.lastCoreConnection > isolationThresholdMs;
+        });
+        // Find orphaned memories (not in any fragment above minClusterSize)
+        const fragmentedIds = new Set();
+        for (const frag of fragments) {
+            if (frag.size < cfg.minClusterSize) {
+                for (const id of frag.memoryIds) {
+                    fragmentedIds.add(id);
+                }
+            }
+        }
+        const orphanedMemories = Array.from(fragmentedIds);
+        // Calculate overall fragmentation score
+        const sizeScore = Math.min(1, fragments.length / Math.max(1, memoryIds.length / cfg.maxHealthyFragmentSize));
+        const coherenceScores = fragments.map((f) => f.internalCoherence);
+        const avgCoherence = coherenceScores.length > 0
+            ? coherenceScores.reduce((a, b) => a + b, 0) / coherenceScores.length
+            : 1;
+        const isolationScore = isolatedFragments.length / Math.max(1, fragments.length);
+        const fragmentationScore = (sizeScore + (1 - avgCoherence) + isolationScore) / 3;
+        avgFragmentationScore = (avgFragmentationScore * (totalAnalyzed - 1) + fragmentationScore) / totalAnalyzed;
+        // Generate recommendations
+        const recommendations = [];
+        if (isolatedFragments.length > 0) {
+            recommendations.push(`${isolatedFragments.length} fragment(s) have no recent connection to core memory. ` +
+                `Consider creating bridge memories that link these to core themes.`);
+        }
+        if (fragmentationScore > 0.6) {
+            recommendations.push(`High fragmentation detected. Try to find semantic connections between fragments ` +
+                `and create linking memories.`);
+        }
+        if (orphanedMemories.length > 0) {
+            recommendations.push(`${orphanedMemories.length} memory/ies are isolated. ` +
+                `Consider connecting them to related fragments or the core memory tier.`);
+        }
+        const lowCoherenceFragments = fragments.filter((f) => f.internalCoherence < cfg.minHealthyCoherence);
+        if (lowCoherenceFragments.length > 0) {
+            recommendations.push(`${lowCoherenceFragments.length} fragment(s) have low internal coherence. ` +
+                `Review if these memories truly belong together or should be reorganized.`);
+        }
+        if (recommendations.length === 0) {
+            recommendations.push('Memory structure is healthy. No immediate consolidation needed.');
+        }
+        logger.info(`Fragmentation analysis complete: ${fragments.length} fragments, score=${fragmentationScore.toFixed(3)}`);
+        return {
+            fragments,
+            isolatedFragments,
+            orphanedMemories,
+            overallFragmentationScore: fragmentationScore,
+            recommendations,
+            timestamp: lastAnalysis,
+        };
+    }
+    function findFragment(memoryId, report) {
+        for (const fragment of report.fragments) {
+            if (fragment.memoryIds.includes(memoryId)) {
+                return fragment;
+            }
+        }
+        return null;
+    }
+    function suggestConnections(fragmentId, report) {
+        const fragment = report.fragments.find((f) => f.id === fragmentId);
+        if (!fragment)
+            return [];
+        // Find fragments with similar themes
+        const similarFragments = report.fragments.filter((f) => {
+            if (f.id === fragmentId)
+                return false;
+            // Check theme overlap
+            const themeA = fragment.theme.toLowerCase();
+            const themeB = f.theme.toLowerCase();
+            const hasOverlap = themeA.split(' + ').some((w) => themeB.includes(w)) ||
+                themeB.split(' + ').some((w) => themeA.includes(w));
+            return hasOverlap;
+        });
+        // Return memory IDs from similar fragments
+        const suggestions = [];
+        for (const similar of similarFragments.slice(0, 3)) {
+            suggestions.push(...similar.memoryIds.slice(0, 3));
+        }
+        return suggestions;
+    }
+    function calculateCoherence(memoryIds, getNode) {
+        if (memoryIds.length <= 1)
+            return 1.0;
+        // Count actual connections within this group
+        let actualConnections = 0;
+        let possibleConnections = (memoryIds.length * (memoryIds.length - 1)) / 2;
+        for (const id of memoryIds) {
+            const node = getNode(id);
+            if (!node)
+                continue;
+            for (const connId of node.connections) {
+                if (memoryIds.includes(connId)) {
+                    actualConnections++;
+                }
+            }
+        }
+        // Each edge counted twice in the loop above
+        actualConnections = actualConnections / 2;
+        return possibleConnections > 0 ? actualConnections / possibleConnections : 0;
+    }
+    function generateBridge(fragmentA, fragmentB) {
+        const themeA = fragmentA.theme;
+        const themeB = fragmentB.theme;
+        const bridge = `Bridge memory: connecting "${themeA}" and "${themeB}" themes. ` +
+            `This memory links ${fragmentA.memoryIds.length} memories from fragment A ` +
+            `with ${fragmentB.memoryIds.length} memories from fragment B. ` +
+            `Key connection: shared semantic elements between ${themeA} and ${themeB}.`;
+        logger.info(`Generated bridge between fragments: ${fragmentA.id} <-> ${fragmentB.id}`);
+        return bridge;
+    }
+    function getStats() {
+        return {
+            totalAnalyzed,
+            avgFragmentationScore,
+            lastAnalysis,
+        };
+    }
+    return {
+        analyze,
+        findFragment,
+        suggestConnections,
+        calculateCoherence,
+        generateBridge,
+        getStats,
+    };
+}

package/dist/core/memory/index.js

@@ -7,3 +7,4 @@ export * from './dream-consolidation.js';
 export * from './spaced-repetition.js';
 export * from './hopfield-network.js';
 export * from './adaptive-rag.js';
+export { createContextFragmentationEngine } from './context-fragmentation.js';