mark-improving-agent 2.2.3 → 2.2.5

package/dist/core/memory/hybrid-search.js

@@ -0,0 +1,177 @@
+/**
+ * Hybrid Memory Search
+ *
+ * Combines vector similarity search with BM25 keyword matching and knowledge graph traversal
+ * Based on Dakera AI's hybrid search architecture (87.8% LoCoMo accuracy)
+ *
+ * @module core/memory
+ * @fileoverview Hybrid search combining vector + BM25 + knowledge graph for superior recall
+ */
+import { createLogger } from '../../utils/logger.js';
+const logger = createLogger('HybridSearch');
+/**
+ * Default hybrid search configuration
+ */
+export const DEFAULT_HYBRID_CONFIG = {
+    vectorWeight: 0.5,
+    bm25Weight: 0.3,
+    kgWeight: 0.2,
+    bm25: { k1: 1.5, b: 0.75 },
+    minScore: 0.1,
+    maxResults: 20,
+};
+/**
+ * Tokenizer for BM25
+ */
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\w\s]/g, ' ')
+        .split(/\s+/)
+        .filter(token => token.length > 2);
+}
+/**
+ * Create a BM25 index from memory entries
+ */
+export function createBM25Index(entries) {
+    const index = {
+        docLengths: new Map(),
+        termDocFreq: new Map(),
+        avgDocLength: 0,
+        totalDocs: entries.length,
+        invertedIndex: new Map(),
+    };
+    let totalLength = 0;
+    for (const entry of entries) {
+        const tokens = tokenize(entry.content);
+        const docLength = tokens.length;
+        index.docLengths.set(entry.id, docLength);
+        totalLength += docLength;
+        // Count term frequencies
+        const termFreq = new Map();
+        for (const token of tokens) {
+            termFreq.set(token, (termFreq.get(token) ?? 0) + 1);
+            if (!index.invertedIndex.has(token)) {
+                index.invertedIndex.set(token, new Map());
+            }
+            const posting = index.invertedIndex.get(token);
+            posting.set(entry.id, termFreq.get(token));
+        }
+        // Update document frequency
+        for (const token of new Set(tokens)) {
+            index.termDocFreq.set(token, (index.termDocFreq.get(token) ?? 0) + 1);
+        }
+    }
+    index.avgDocLength = totalLength / Math.max(entries.length, 1);
+    return index;
+}
+/**
+ * Calculate BM25 score for a single document
+ */
+export function bm25Score(index, docId, queryTokens, config) {
+    const docLength = index.docLengths.get(docId) ?? 0;
+    let score = 0;
+    for (const token of queryTokens) {
+        const tf = index.invertedIndex.get(token)?.get(docId) ?? 0;
+        if (tf === 0)
+            continue;
+        const df = index.termDocFreq.get(token) ?? 0;
+        if (df === 0)
+            continue;
+        const idf = Math.log((index.totalDocs - df + 0.5) / (df + 0.5) + 1);
+        const tfComponent = (tf * (config.k1 + 1)) / (tf + config.k1 * (1 - config.b + config.b * (docLength / index.avgDocLength)));
+        score += idf * tfComponent;
+    }
+    return score;
+}
+/**
+ * Normalize BM25 scores to 0-1 range
+ */
+export function normalizeBM25Scores(scores) {
+    const maxScore = Math.max(...Array.from(scores.values()), 1);
+    const normalized = new Map();
+    for (const [docId, score] of scores) {
+        normalized.set(docId, score / maxScore);
+    }
+    return normalized;
+}
+/**
+ * Create a hybrid search engine
+ */
+export function createHybridSearchEngine(initialEntries = [], config = {}) {
+    const fullConfig = {
+        ...DEFAULT_HYBRID_CONFIG,
+        ...config,
+        bm25: { ...DEFAULT_HYBRID_CONFIG.bm25, ...config.bm25 },
+    };
+    let bm25Index = createBM25Index(initialEntries);
+    function rebuildIndex(entries) {
+        logger.info(`Rebuilding BM25 index with ${entries.length} documents`);
+        bm25Index = createBM25Index(entries);
+    }
+    function search(query, entries, options) {
+        const searchConfig = {
+            ...fullConfig,
+            ...options?.config,
+        };
+        // Tokenize query for BM25
+        const queryTokens = tokenize(query);
+        // Calculate BM25 scores for all entries
+        const bm25Scores = new Map();
+        for (const entry of entries) {
+            const score = bm25Score(bm25Index, entry.id, queryTokens, searchConfig.bm25);
+            if (score > 0) {
+                bm25Scores.set(entry.id, score);
+            }
+        }
+        const normalizedBM25 = normalizeBM25Scores(bm25Scores);
+        // Calculate KG scores based on connection count
+        const kgScores = new Map();
+        if (options?.kgConnections) {
+            const maxConnections = Math.max(...Array.from(options.kgConnections.values()).map(arr => arr.length), 1);
+            for (const entry of entries) {
+                const connections = options.kgConnections.get(entry.id)?.length ?? 0;
+                kgScores.set(entry.id, connections / maxConnections);
+            }
+        }
+        // Combine scores
+        const combinedResults = new Map();
+        for (const entry of entries) {
+            const vectorScore = options?.vectorScores?.get(entry.id) ?? 0;
+            const bm25Score = normalizedBM25.get(entry.id) ?? 0;
+            const kgScore = kgScores.get(entry.id) ?? 0;
+            // Weighted combination
+            const combinedScore = (vectorScore * searchConfig.vectorWeight) +
+                (bm25Score * searchConfig.bm25Weight) +
+                (kgScore * searchConfig.kgWeight);
+            if (combinedScore >= searchConfig.minScore) {
+                combinedResults.set(entry.id, {
+                    entry,
+                    score: combinedScore,
+                    combinedScore,
+                    vectorScore,
+                    bm25Score,
+                    kgScore,
+                    reason: 'semantic',
+                });
+            }
+        }
+        // Sort by combined score
+        const results = Array.from(combinedResults.values())
+            .sort((a, b) => b.combinedScore - a.combinedScore)
+            .slice(0, searchConfig.maxResults);
+        logger.debug(`Hybrid search for "${query}": ${results.length} results`);
+        return results;
+    }
+    function getStats() {
+        return {
+            indexedDocs: bm25Index.totalDocs,
+            avgDocLength: Math.round(bm25Index.avgDocLength),
+        };
+    }
+    return {
+        search,
+        rebuildIndex,
+        getStats,
+    };
+}

package/dist/core/memory/index.js

@@ -7,3 +7,5 @@ export * from './dream-consolidation.js';
 export * from './spaced-repetition.js';
 export * from './hopfield-network.js';
 export * from './adaptive-rag.js';
+export { createContextFragmentationEngine } from './context-fragmentation.js';
+export { createHybridSearchEngine, createBM25Index, bm25Score, normalizeBM25Scores, DEFAULT_HYBRID_CONFIG } from './hybrid-search.js';

package/dist/core/ontology/concept-graph.js

@@ -0,0 +1,437 @@
+/**
+ * Concept Graph - Structured Knowledge Representation
+ *
+ * Builds and maintains a graph of typed concepts extracted from
+ * memory entries. Each concept has structured metadata (frontmatter-
+ * style) and relationships to other concepts.
+ *
+ * Based on oh-my-ontology's approach: markdown frontmatter as graph,
+ * but adapted for AI agent memory systems.
+ *
+ * @module core/ontology
+ * @fileoverview Concept graph implementation
+ */
+import { randomUUID } from 'crypto';
+import { createLogger } from '../../utils/logger.js';
+const logger = createLogger('[ConceptGraph]');
+// ============================================================
+// Constants
+// ============================================================
+const VALID_DOMAINS = [
+    'auth', 'memory', 'cognition', 'collaboration', 'identity',
+    'evolution', 'consciousness', 'psychology', 'security', 'meta'
+];
+const VALID_KINDS = [
+    'entity', 'capability', 'principle', 'pattern', 'process', 'rule'
+];
+const DOMAIN_ALIASES = {
+    'mcp': 'collaboration',
+    'multi-agent': 'collaboration',
+    'agent': 'collaboration',
+    'memory': 'memory',
+    ' episodic': 'memory',
+    'semantic': 'memory',
+    'consciousness': 'consciousness',
+    'identity': 'identity',
+    'self': 'identity',
+    'evolution': 'evolution',
+    'learn': 'evolution',
+    'psychology': 'psychology',
+    'emotion': 'psychology',
+    'security': 'security',
+    'safety': 'security',
+};
+// ============================================================
+// Utility Functions
+// ============================================================
+function normalizeDomain(domain) {
+    const lower = domain.toLowerCase().replace(/[_-]/g, '-');
+    return DOMAIN_ALIASES[lower] || lower;
+}
+function slugify(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-|-$/g, '')
+        .slice(0, 64);
+}
+function extractKeywords(text) {
+    const stopWords = new Set([
+        'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+        'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+        'should', 'may', 'might', 'must', 'can', 'need', 'to', 'of', 'in',
+        'for', 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through',
+        'and', 'or', 'if', 'but', 'so', 'than', 'then', 'when', 'where',
+        'why', 'how', 'what', 'which', 'who', 'whom', 'this', 'that', 'these',
+        'those', 'it', 'its', 'they', 'their', 'them', 'we', 'our', 'us',
+        '你', '我', '他', '她', '它', '的', '是', '在', '了', '和', '与'
+    ]);
+    const words = [];
+    const tokens = text.toLowerCase().split(/\s+/);
+    for (const token of tokens) {
+        const cleaned = token.replace(/[^a-zA-Z\u4e00-\u9fff]/g, '');
+        if (cleaned.length >= 3 && !stopWords.has(cleaned)) {
+            words.push(cleaned);
+        }
+    }
+    return [...new Set(words)].slice(0, 10);
+}
+const DEFAULT_OPTIONS = {
+    minConfidence: 0.5,
+    orphanMaxAge: 7 * 24 * 60 * 60 * 1000, // 7 days
+    autoLink: true,
+};
+export function createConceptGraph(options = {}) {
+    const opts = { ...DEFAULT_OPTIONS, ...options };
+    const concepts = new Map();
+    const conceptsBySlug = new Map(); // key: domain/slug
+    const relations = new Map();
+    const memoryToConcepts = new Map(); // memoryId -> conceptIds
+    function buildSlugKey(domain, slug) {
+        return `${domain}/${slug}`;
+    }
+    function createConcept(data) {
+        const now = Date.now();
+        const domain = normalizeDomain(data.domain);
+        const slug = data.slug || slugify(data.name);
+        // Check for duplicate slug in domain
+        const key = buildSlugKey(domain, slug);
+        if (conceptsBySlug.has(key)) {
+            const existing = conceptsBySlug.get(key);
+            // Merge sources if same slug
+            if (data.sourceMemories) {
+                existing.sourceMemories = [...new Set([...existing.sourceMemories, ...data.sourceMemories])];
+                existing.updatedAt = now;
+            }
+            logger.info(`Merged with existing concept: ${key}`);
+            return existing;
+        }
+        const concept = {
+            id: data.id || randomUUID(),
+            slug,
+            name: data.name,
+            domain,
+            kind: data.kind || 'entity',
+            description: data.description || '',
+            properties: data.properties || {},
+            dependsOn: data.dependsOn || [],
+            dependents: data.dependents || [],
+            sourceMemories: data.sourceMemories || [],
+            confidence: data.confidence || opts.minConfidence,
+            referenceCount: 0,
+            lastReferenced: now,
+            createdAt: now,
+            updatedAt: now,
+        };
+        concepts.set(concept.id, concept);
+        conceptsBySlug.set(key, concept);
+        logger.info(`Created concept: ${key} (${concept.kind})`);
+        return concept;
+    }
+    function getConcept(id) {
+        return concepts.get(id) || null;
+    }
+    function getConceptBySlug(slug, domain) {
+        return conceptsBySlug.get(buildSlugKey(normalizeDomain(domain), slug)) || null;
+    }
+    function updateConcept(id, updates) {
+        const concept = concepts.get(id);
+        if (!concept)
+            return null;
+        // Update slug if name/domain changed
+        if (updates.domain && updates.domain !== concept.domain) {
+            const oldKey = buildSlugKey(concept.domain, concept.slug);
+            conceptsBySlug.delete(oldKey);
+            concept.domain = normalizeDomain(updates.domain);
+        }
+        if (updates.name && updates.name !== concept.name) {
+            const oldKey = buildSlugKey(concept.domain, concept.slug);
+            conceptsBySlug.delete(oldKey);
+            concept.slug = slugify(updates.name);
+        }
+        Object.assign(concept, updates, { updatedAt: Date.now() });
+        const newKey = buildSlugKey(concept.domain, concept.slug);
+        conceptsBySlug.set(newKey, concept);
+        logger.info(`Updated concept: ${concept.id}`);
+        return concept;
+    }
+    function deleteConcept(id) {
+        const concept = concepts.get(id);
+        if (!concept)
+            return false;
+        // Remove from slug index
+        const key = buildSlugKey(concept.domain, concept.slug);
+        conceptsBySlug.delete(key);
+        // Remove relations involving this concept
+        for (const [relId, rel] of relations) {
+            if (rel.sourceId === id || rel.targetId === id) {
+                relations.delete(relId);
+            }
+        }
+        // Remove from memory links
+        for (const [, conceptIds] of memoryToConcepts) {
+            conceptIds.delete(id);
+        }
+        // Update dependents of concepts this depended on
+        for (const depId of concept.dependsOn) {
+            const dep = concepts.get(depId);
+            if (dep) {
+                dep.dependents = dep.dependents.filter(d => d !== id);
+            }
+        }
+        concepts.delete(id);
+        logger.info(`Deleted concept: ${id}`);
+        return true;
+    }
+    function createRelation(sourceId, targetId, relation, weight = 0.8) {
+        const source = concepts.get(sourceId);
+        const target = concepts.get(targetId);
+        if (!source || !target) {
+            logger.warn(`Cannot create relation: concept not found`);
+            return null;
+        }
+        // Check for duplicate
+        for (const rel of relations.values()) {
+            if (rel.sourceId === sourceId && rel.targetId === targetId && rel.relation === relation) {
+                return rel;
+            }
+        }
+        const rel = {
+            id: randomUUID(),
+            sourceId,
+            targetId,
+            relation,
+            weight,
+            bidirectional: ['associates', 'part-of'].includes(relation),
+            createdAt: Date.now(),
+        };
+        relations.set(rel.id, rel);
+        // Update concept dependency lists
+        if (!source.dependsOn.includes(targetId) && (relation === 'depends-on' || relation === 'enables')) {
+            source.dependsOn.push(targetId);
+        }
+        if (!target.dependents.includes(sourceId)) {
+            target.dependents.push(sourceId);
+        }
+        logger.info(`Created relation: ${source.slug} --[${relation}]--> ${target.slug}`);
+        return rel;
+    }
+    function deleteRelation(relationId) {
+        const rel = relations.get(relationId);
+        if (!rel)
+            return false;
+        // Update concept dependency lists
+        const source = concepts.get(rel.sourceId);
+        const target = concepts.get(rel.targetId);
+        if (source) {
+            source.dependsOn = source.dependsOn.filter(id => id !== rel.targetId);
+        }
+        if (target) {
+            target.dependents = target.dependents.filter(id => id !== rel.sourceId);
+        }
+        relations.delete(relationId);
+        return true;
+    }
+    function getConceptRelations(conceptId) {
+        const result = [];
+        for (const rel of relations.values()) {
+            if (rel.sourceId === conceptId || rel.targetId === conceptId) {
+                result.push(rel);
+            }
+        }
+        return result;
+    }
+    function search(query) {
+        const results = [];
+        const now = Date.now();
+        for (const concept of concepts.values()) {
+            let score = 0;
+            // Domain filter
+            if (query.domain) {
+                if (concept.domain !== normalizeDomain(query.domain))
+                    continue;
+                score += 0.3;
+            }
+            // Kind filter
+            if (query.kind) {
+                if (concept.kind !== query.kind)
+                    continue;
+                score += 0.2;
+            }
+            // Slug exact match
+            if (query.slug) {
+                if (concept.slug !== query.slug)
+                    continue;
+                score += 0.4;
+            }
+            // Text search
+            if (query.search) {
+                const searchLower = query.search.toLowerCase();
+                const searchTerms = extractKeywords(query.search);
+                if (concept.name.toLowerCase().includes(searchLower)) {
+                    score += 0.3;
+                }
+                if (concept.description.toLowerCase().includes(searchLower)) {
+                    score += 0.2;
+                }
+                if (searchTerms.some(term => {
+                    const descLower = concept.description.toLowerCase();
+                    const propStr = JSON.stringify(concept.properties).toLowerCase();
+                    return descLower.includes(term) || propStr.includes(term);
+                })) {
+                    score += 0.15;
+                }
+            }
+            // Confidence filter
+            if (query.minConfidence && concept.confidence < query.minConfidence) {
+                continue;
+            }
+            if (score > 0) {
+                results.push({ concept, score });
+            }
+        }
+        // Sort by score, then by reference count and recency
+        results.sort((a, b) => {
+            if (b.score !== a.score)
+                return b.score - a.score;
+            const aEngagement = a.concept.referenceCount * (1 / Math.max(1, now - a.concept.lastReferenced));
+            const bEngagement = b.concept.referenceCount * (1 / Math.max(1, now - b.concept.lastReferenced));
+            return bEngagement - aEngagement;
+        });
+        const limit = query.limit || 20;
+        return results.slice(0, limit).map(r => r.concept);
+    }
+    function infer(conceptId, depth = 2) {
+        const rootConcept = concepts.get(conceptId);
+        if (!rootConcept)
+            return [];
+        const results = [];
+        const visited = new Set([conceptId]);
+        function traverse(currentId, path, currentDepth, currentConcept) {
+            if (currentDepth >= depth)
+                return;
+            const relations = getConceptRelations(currentId);
+            for (const rel of relations) {
+                const neighborId = rel.sourceId === currentId ? rel.targetId : rel.sourceId;
+                if (visited.has(neighborId))
+                    continue;
+                visited.add(neighborId);
+                const neighbor = concepts.get(neighborId);
+                if (!neighbor)
+                    continue;
+                const newPath = [...path, neighbor.slug];
+                const reasoning = `Through ${rel.relation} relation from ${path[path.length - 1] || currentConcept.slug}`;
+                results.push({
+                    path: newPath,
+                    inferredConcept: neighbor,
+                    confidence: currentConcept.confidence * rel.weight * Math.pow(0.9, currentDepth),
+                    reasoning,
+                });
+                traverse(neighborId, newPath, currentDepth + 1, neighbor);
+            }
+        }
+        traverse(conceptId, [rootConcept.slug], 0, rootConcept);
+        return results.sort((a, b) => b.confidence - a.confidence);
+    }
+    function referenceConcept(conceptId) {
+        const concept = concepts.get(conceptId);
+        if (concept) {
+            concept.referenceCount++;
+            concept.lastReferenced = Date.now();
+        }
+    }
+    function linkMemoryToConcepts(memoryId, conceptIds) {
+        memoryToConcepts.set(memoryId, new Set(conceptIds));
+        for (const cid of conceptIds) {
+            const concept = concepts.get(cid);
+            if (concept && !concept.sourceMemories.includes(memoryId)) {
+                concept.sourceMemories.push(memoryId);
+            }
+        }
+    }
+    function getOrphanConcepts() {
+        const now = Date.now();
+        const orphans = [];
+        for (const concept of concepts.values()) {
+            const relations = getConceptRelations(concept.id);
+            const hasConnections = relations.length > 0 || concept.dependsOn.length > 0 || concept.dependents.length > 0;
+            const isStale = now - concept.lastReferenced > opts.orphanMaxAge;
+            if (!hasConnections || isStale) {
+                orphans.push(concept);
+            }
+        }
+        return orphans;
+    }
+    function getStats() {
+        const now = Date.now();
+        const domains = new Set();
+        let totalConnections = 0;
+        let mostConnected = [];
+        const orphanConcepts = getOrphanConcepts();
+        const recentlyReferenced = [];
+        for (const concept of concepts.values()) {
+            domains.add(concept.domain);
+            const rels = getConceptRelations(concept.id);
+            concept.referenceCount = rels.length + concept.dependsOn.length + concept.dependents.length;
+            totalConnections += rels.length;
+            if (now - concept.lastReferenced < 24 * 60 * 60 * 1000) {
+                recentlyReferenced.push(concept);
+            }
+        }
+        mostConnected = [...concepts.values()]
+            .sort((a, b) => b.referenceCount - a.referenceCount)
+            .slice(0, 5);
+        return {
+            totalConcepts: concepts.size,
+            totalRelations: relations.size,
+            domains: [...domains].sort(),
+            avgConnections: concepts.size > 0 ? totalConnections / concepts.size : 0,
+            mostConnected,
+            orphanConcepts,
+            recentlyReferenced: recentlyReferenced.sort((a, b) => b.lastReferenced - a.lastReferenced),
+        };
+    }
+    function exportData() {
+        return {
+            concepts: [...concepts.values()],
+            relations: [...relations.values()],
+        };
+    }
+    function importData(data) {
+        clear();
+        for (const concept of data.concepts) {
+            concepts.set(concept.id, concept);
+            conceptsBySlug.set(buildSlugKey(concept.domain, concept.slug), concept);
+        }
+        for (const rel of data.relations) {
+            relations.set(rel.id, rel);
+        }
+        logger.info(`Imported ${data.concepts.length} concepts, ${data.relations.length} relations`);
+    }
+    function clear() {
+        concepts.clear();
+        conceptsBySlug.clear();
+        relations.clear();
+        memoryToConcepts.clear();
+        logger.info('Cleared concept graph');
+    }
+    return {
+        createConcept,
+        getConcept,
+        getConceptBySlug,
+        updateConcept,
+        deleteConcept,
+        createRelation,
+        deleteRelation,
+        getConceptRelations,
+        search,
+        infer,
+        referenceConcept,
+        linkMemoryToConcepts,
+        getOrphanConcepts,
+        getStats,
+        exportData,
+        importData,
+        clear,
+    };
+}

package/dist/core/ontology/index.js

@@ -0,0 +1,18 @@
+/**
+ * Concept Ontology System
+ *
+ * Structured knowledge representation using typed concepts
+ * and relationships, inspired by oh-my-ontology's frontmatter
+ * graph approach. Enables AI-native concept reasoning.
+ *
+ * Key mechanisms:
+ * - Concept nodes with typed properties
+ * - Relationship edges with semantic meaning
+ * - Cross-reference resolution
+ * - Concept hierarchy (is-a, part-of)
+ * - Inference through relation traversal
+ *
+ * @module core/ontology
+ * @fileoverview Concept ontology for structured knowledge
+ */
+export * from './concept-graph.js';

package/dist/index.js

@@ -4,5 +4,8 @@ export { VERSION } from './version.js';
 export * from './core/psychology/index.js';
 export * from './core/cognition/index.js';
 export { createAdaptiveRAGMemory, formatRetrievalResult } from './core/memory/adaptive-rag.js';
+export { createContextFragmentationEngine } from './core/memory/context-fragmentation.js';
+export { createIdentityContinuityVerifier } from './core/identity/identity-continuity.js';
+export { createMCPProtocol } from './core/collaboration/mcp-protocol.js';
 export { createTruthTeller, formatTruthStatement } from './core/truth-teller.js';
 export { createActiveInferenceEngine, formatFreeEnergyMetrics, formatBeliefState } from './core/cognition/active-inference.js';

package/dist/version.js

@@ -1 +1 @@
-export const VERSION = '2.2.3';
+export const VERSION = '2.2.5';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mark-improving-agent",
-  "version": "2.2.3",
+  "version": "2.2.5",
   "description": "Self-evolving AI agent with permanent memory, identity continuity, and self-evolution — for AI agents that need to remember, learn, and evolve across sessions",
   "type": "module",
   "main": "./dist/index.js",