@soulcraft/brainy 3.9.1 → 3.10.0

package/dist/vfs/ConceptSystem.js

@@ -0,0 +1,598 @@
+/**
+ * Universal Concept System for VFS
+ *
+ * Manages concepts that transcend files and exist independently
+ * Ideas that can be linked to multiple manifestations across domains
+ * PRODUCTION-READY: Real implementation using Brainy
+ */
+import { NounType, VerbType } from '../types/graphTypes.js';
+import { cosineDistance } from '../utils/distance.js';
+import { v4 as uuidv4 } from '../universal/uuid.js';
+/**
+ * Universal Concept System
+ *
+ * Manages concepts that exist independently of any specific file or context
+ * Examples:
+ * - "Authentication" concept appearing in docs, code, tests
+ * - "Customer Journey" concept in marketing, UX, analytics
+ * - "Dependency Injection" pattern across multiple codebases
+ * - "Sustainability" theme in various research papers
+ */
+export class ConceptSystem {
+    constructor(brain, config) {
+        this.brain = brain;
+        this.conceptCache = new Map();
+        this.config = {
+            autoLink: config?.autoLink ?? false,
+            similarityThreshold: config?.similarityThreshold ?? 0.7,
+            maxManifestations: config?.maxManifestations ?? 1000,
+            strengthDecay: config?.strengthDecay ?? 0.95 // 5% decay over time
+        };
+    }
+    /**
+     * Create a new universal concept
+     */
+    async createConcept(concept) {
+        const conceptId = uuidv4();
+        const timestamp = Date.now();
+        const universalConcept = {
+            ...concept,
+            id: conceptId,
+            created: timestamp,
+            lastUpdated: timestamp,
+            version: 1,
+            links: [],
+            manifestations: []
+        };
+        // Generate embedding for concept
+        let embedding;
+        try {
+            embedding = await this.generateConceptEmbedding(universalConcept);
+        }
+        catch (error) {
+            console.warn('Failed to generate concept embedding:', error);
+        }
+        // Store concept in Brainy
+        const brainyEntity = await this.brain.add({
+            type: NounType.Concept,
+            data: Buffer.from(JSON.stringify(universalConcept)),
+            metadata: {
+                ...universalConcept,
+                conceptType: 'universal',
+                system: 'vfs-concept'
+            },
+            vector: embedding
+        });
+        // Auto-link to similar concepts if enabled
+        if (this.config.autoLink) {
+            await this.autoLinkConcept(conceptId);
+        }
+        // Update cache
+        this.conceptCache.set(conceptId, universalConcept);
+        return brainyEntity;
+    }
+    /**
+     * Find concepts by various criteria
+     */
+    async findConcepts(query) {
+        const searchQuery = {
+            conceptType: 'universal',
+            system: 'vfs-concept'
+        };
+        // Direct attribute matching
+        if (query.name)
+            searchQuery.name = query.name;
+        if (query.domain)
+            searchQuery.domain = query.domain;
+        if (query.category)
+            searchQuery.category = query.category;
+        // Keyword matching
+        if (query.keywords && query.keywords.length > 0) {
+            searchQuery.keywords = { $in: query.keywords };
+        }
+        // File manifestation search
+        if (query.manifestedIn) {
+            // Find concepts that have manifestations in this file
+            const manifestationResults = await this.brain.find({
+                where: {
+                    filePath: query.manifestedIn,
+                    eventType: 'concept-manifestation',
+                    system: 'vfs-concept'
+                },
+                type: NounType.Event,
+                limit: 1000
+            });
+            const conceptIds = manifestationResults.map(r => r.entity.metadata.conceptId);
+            if (conceptIds.length > 0) {
+                searchQuery.id = { $in: conceptIds };
+            }
+            else {
+                return []; // No concepts found in this file
+            }
+        }
+        // Search in Brainy
+        let results = await this.brain.find({
+            where: searchQuery,
+            type: NounType.Concept,
+            limit: 1000
+        });
+        // If searching for similar concepts, use vector similarity
+        if (query.similar) {
+            try {
+                const queryEmbedding = await this.generateTextEmbedding(query.similar);
+                if (queryEmbedding) {
+                    // Get all concepts and rank by similarity
+                    const allConcepts = await this.brain.find({
+                        where: { conceptType: 'universal', system: 'vfs-concept' },
+                        type: NounType.Concept,
+                        limit: 10000
+                    });
+                    const withSimilarity = allConcepts
+                        .filter(c => c.entity.vector && c.entity.vector.length > 0)
+                        .map(c => ({
+                        concept: c,
+                        similarity: 1 - cosineDistance(queryEmbedding, c.entity.vector)
+                    }))
+                        .filter(s => s.similarity > this.config.similarityThreshold)
+                        .sort((a, b) => b.similarity - a.similarity);
+                    results = withSimilarity.map(s => s.concept);
+                }
+            }
+            catch (error) {
+                console.warn('Failed to perform concept similarity search:', error);
+            }
+        }
+        return results.map(r => r.entity.metadata);
+    }
+    /**
+     * Link two concepts together
+     */
+    async linkConcept(fromConceptId, toConceptId, relationship, options) {
+        const linkId = uuidv4();
+        const fromConcept = await this.getConcept(fromConceptId);
+        const toConcept = await this.getConcept(toConceptId);
+        if (!fromConcept || !toConcept) {
+            throw new Error('One or both concepts not found');
+        }
+        // Create link
+        const link = {
+            id: linkId,
+            targetConceptId: toConceptId,
+            relationship,
+            strength: options?.strength ?? 0.8,
+            context: options?.context,
+            bidirectional: options?.bidirectional ?? false
+        };
+        // Add link to source concept
+        fromConcept.links.push(link);
+        fromConcept.lastUpdated = Date.now();
+        await this.updateConcept(fromConcept);
+        // Add bidirectional link if specified
+        if (link.bidirectional) {
+            const reverseRelationship = this.getReverseRelationship(relationship);
+            const reverseLink = {
+                id: uuidv4(),
+                targetConceptId: fromConceptId,
+                relationship: reverseRelationship,
+                strength: link.strength,
+                context: link.context,
+                bidirectional: true
+            };
+            toConcept.links.push(reverseLink);
+            toConcept.lastUpdated = Date.now();
+            await this.updateConcept(toConcept);
+        }
+        // Create Brainy relationship
+        await this.brain.relate({
+            from: fromConceptId,
+            to: toConceptId,
+            type: this.getVerbType(relationship),
+            metadata: {
+                strength: link.strength,
+                context: link.context,
+                bidirectional: link.bidirectional
+            }
+        });
+        return linkId;
+    }
+    /**
+     * Record a manifestation of a concept in a file
+     */
+    async recordManifestation(conceptId, filePath, context, form, options) {
+        const concept = await this.getConcept(conceptId);
+        if (!concept) {
+            throw new Error(`Concept ${conceptId} not found`);
+        }
+        const manifestationId = uuidv4();
+        const timestamp = Date.now();
+        const manifestation = {
+            id: manifestationId,
+            conceptId,
+            filePath,
+            context,
+            form,
+            position: options?.position,
+            confidence: options?.confidence ?? 1.0,
+            timestamp,
+            extractedBy: options?.extractedBy ?? 'manual'
+        };
+        // Store manifestation as Brainy event
+        await this.brain.add({
+            type: NounType.Event,
+            data: Buffer.from(context),
+            metadata: {
+                ...manifestation,
+                eventType: 'concept-manifestation',
+                system: 'vfs-concept'
+            }
+        });
+        // Create relationship to concept
+        await this.brain.relate({
+            from: manifestationId,
+            to: conceptId,
+            type: VerbType.Implements
+        });
+        // Update concept with new manifestation
+        concept.manifestations.push(manifestation);
+        concept.lastUpdated = timestamp;
+        // Update concept strength based on manifestations
+        concept.strength = Math.min(1.0, concept.strength + 0.1);
+        // Prune old manifestations if needed
+        if (concept.manifestations.length > this.config.maxManifestations) {
+            concept.manifestations = concept.manifestations
+                .sort((a, b) => b.timestamp - a.timestamp)
+                .slice(0, this.config.maxManifestations);
+        }
+        // Update stored concept
+        await this.updateConcept(concept);
+        return manifestationId;
+    }
+    /**
+     * Extract and link concepts from content
+     */
+    async extractAndLinkConcepts(filePath, content) {
+        if (!this.config.autoLink) {
+            return [];
+        }
+        const text = content.toString('utf8');
+        const extractedConcepts = [];
+        // Simple concept extraction patterns
+        // In production, this would use advanced NLP/AI models
+        const conceptPatterns = [
+            // Technical concepts
+            /\b(authentication|authorization|validation|encryption|caching|logging|monitoring)\b/gi,
+            // Business concepts
+            /\b(customer\s+journey|user\s+experience|business\s+logic|revenue\s+model)\b/gi,
+            // Design patterns
+            /\b(singleton|factory|observer|strategy|adapter|decorator)\b/gi,
+            // General concepts
+            /\b(security|performance|scalability|maintainability|reliability)\b/gi
+        ];
+        for (const pattern of conceptPatterns) {
+            const matches = text.matchAll(pattern);
+            for (const match of matches) {
+                const conceptName = match[0].toLowerCase();
+                const context = this.extractContext(text, match.index || 0);
+                // Find or create concept
+                let concepts = await this.findConcepts({ name: conceptName });
+                let conceptId;
+                if (concepts.length === 0) {
+                    // Create new concept
+                    conceptId = await this.createConcept({
+                        name: conceptName,
+                        domain: this.detectDomain(conceptName, text),
+                        category: this.detectCategory(conceptName),
+                        keywords: [conceptName],
+                        strength: 0.5,
+                        metadata: {}
+                    });
+                    extractedConcepts.push(conceptId);
+                }
+                else {
+                    conceptId = concepts[0].id;
+                }
+                // Record manifestation
+                await this.recordManifestation(conceptId, filePath, context, this.detectManifestationForm(context), {
+                    confidence: 0.8,
+                    extractedBy: 'auto'
+                });
+            }
+        }
+        return extractedConcepts;
+    }
+    /**
+     * Get concept graph for visualization
+     */
+    async getConceptGraph(options) {
+        const query = {
+            conceptType: 'universal',
+            system: 'vfs-concept'
+        };
+        if (options?.domain) {
+            query.domain = options.domain;
+        }
+        if (options?.minStrength) {
+            query.strength = { $gte: options.minStrength };
+        }
+        const results = await this.brain.find({
+            where: query,
+            type: NounType.Concept,
+            limit: options?.maxConcepts || 1000
+        });
+        const concepts = results.map(r => r.entity.metadata);
+        // Build graph structure
+        const graphConcepts = concepts.map(c => ({
+            id: c.id,
+            name: c.name,
+            domain: c.domain,
+            strength: c.strength,
+            manifestationCount: c.manifestations.length
+        }));
+        const graphLinks = [];
+        for (const concept of concepts) {
+            for (const link of concept.links) {
+                // Only include links to concepts in our result set
+                if (concepts.find(c => c.id === link.targetConceptId)) {
+                    graphLinks.push({
+                        source: concept.id,
+                        target: link.targetConceptId,
+                        relationship: link.relationship,
+                        strength: link.strength
+                    });
+                }
+            }
+        }
+        return {
+            concepts: graphConcepts,
+            links: graphLinks
+        };
+    }
+    /**
+     * Find appearances of a concept
+     */
+    async findAppearances(conceptId, options) {
+        const query = {
+            conceptId,
+            eventType: 'concept-manifestation',
+            system: 'vfs-concept'
+        };
+        if (options?.filePath) {
+            query.filePath = options.filePath;
+        }
+        if (options?.form) {
+            query.form = options.form;
+        }
+        if (options?.minConfidence) {
+            query.confidence = { $gte: options.minConfidence };
+        }
+        const results = await this.brain.find({
+            where: query,
+            type: NounType.Event,
+            limit: options?.limit || 1000
+        });
+        return results
+            .map(r => r.entity.metadata)
+            .sort((a, b) => b.timestamp - a.timestamp);
+    }
+    /**
+     * Auto-link concept to similar concepts
+     */
+    async autoLinkConcept(conceptId) {
+        const concept = await this.getConcept(conceptId);
+        if (!concept)
+            return;
+        // Find similar concepts
+        const similar = await this.findConcepts({
+            similar: concept.name + ' ' + (concept.description || '')
+        });
+        for (const similarConcept of similar) {
+            if (similarConcept.id === conceptId)
+                continue;
+            // Calculate relationship strength based on similarity
+            const strength = await this.calculateConceptSimilarity(concept, similarConcept);
+            if (strength > this.config.similarityThreshold) {
+                await this.linkConcept(conceptId, similarConcept.id, 'related', { strength, bidirectional: true });
+            }
+        }
+    }
+    /**
+     * Get concept by ID
+     */
+    async getConcept(conceptId) {
+        // Check cache first
+        if (this.conceptCache.has(conceptId)) {
+            return this.conceptCache.get(conceptId);
+        }
+        // Query from Brainy
+        const results = await this.brain.find({
+            where: {
+                id: conceptId,
+                conceptType: 'universal',
+                system: 'vfs-concept'
+            },
+            type: NounType.Concept,
+            limit: 1
+        });
+        if (results.length === 0) {
+            return null;
+        }
+        const concept = results[0].entity.metadata;
+        this.conceptCache.set(conceptId, concept);
+        return concept;
+    }
+    /**
+     * Update stored concept
+     */
+    async updateConcept(concept) {
+        // Find the Brainy entity
+        const results = await this.brain.find({
+            where: {
+                id: concept.id,
+                conceptType: 'universal',
+                system: 'vfs-concept'
+            },
+            type: NounType.Concept,
+            limit: 1
+        });
+        if (results.length > 0) {
+            await this.brain.update({
+                id: results[0].entity.id,
+                data: Buffer.from(JSON.stringify(concept)),
+                metadata: {
+                    ...concept,
+                    conceptType: 'universal',
+                    system: 'vfs-concept'
+                }
+            });
+        }
+        // Update cache
+        this.conceptCache.set(concept.id, concept);
+    }
+    /**
+     * Calculate similarity between two concepts
+     */
+    async calculateConceptSimilarity(concept1, concept2) {
+        // Simple similarity calculation
+        let similarity = 0;
+        // Domain similarity
+        if (concept1.domain === concept2.domain)
+            similarity += 0.3;
+        // Category similarity
+        if (concept1.category === concept2.category)
+            similarity += 0.2;
+        // Keyword overlap
+        const commonKeywords = concept1.keywords.filter(k => concept2.keywords.includes(k));
+        similarity += (commonKeywords.length / Math.max(concept1.keywords.length, concept2.keywords.length)) * 0.3;
+        // Name similarity (simple string comparison)
+        const nameWords1 = concept1.name.toLowerCase().split(/\s+/);
+        const nameWords2 = concept2.name.toLowerCase().split(/\s+/);
+        const commonWords = nameWords1.filter(w => nameWords2.includes(w));
+        similarity += (commonWords.length / Math.max(nameWords1.length, nameWords2.length)) * 0.2;
+        return Math.min(1.0, similarity);
+    }
+    /**
+     * Generate embedding for concept
+     */
+    async generateConceptEmbedding(concept) {
+        try {
+            const text = [
+                concept.name,
+                concept.description || '',
+                concept.domain,
+                concept.category,
+                ...(Array.isArray(concept.keywords) ? concept.keywords : [])
+            ].join(' ');
+            return await this.generateTextEmbedding(text);
+        }
+        catch (error) {
+            console.error('Failed to generate concept embedding:', error);
+            return undefined;
+        }
+    }
+    /**
+     * Generate embedding for text
+     */
+    async generateTextEmbedding(text) {
+        try {
+            // Generate embedding using Brainy's embed method
+            const vector = await this.brain.embed(text);
+            return vector;
+        }
+        catch (error) {
+            console.debug('Failed to generate embedding:', error);
+            return undefined;
+        }
+    }
+    /**
+     * Get reverse relationship type
+     */
+    getReverseRelationship(relationship) {
+        const reverseMap = {
+            'extends': 'extended-by',
+            'implements': 'implemented-by',
+            'uses': 'used-by',
+            'opposite': 'opposite',
+            'related': 'related',
+            'contains': 'part-of',
+            'part-of': 'contains'
+        };
+        return reverseMap[relationship] || 'related';
+    }
+    /**
+     * Map concept relationship to VerbType
+     */
+    getVerbType(relationship) {
+        const verbMap = {
+            'extends': VerbType.Extends,
+            'implements': VerbType.Implements,
+            'uses': VerbType.Uses,
+            'opposite': VerbType.Conflicts,
+            'related': VerbType.RelatedTo,
+            'contains': VerbType.Contains,
+            'part-of': VerbType.PartOf
+        };
+        return verbMap[relationship] || VerbType.RelatedTo;
+    }
+    /**
+     * Detect concept domain from context
+     */
+    detectDomain(conceptName, context) {
+        if (/import|export|function|class|const|var|let/.test(context))
+            return 'technical';
+        if (/customer|user|business|revenue|market/.test(context))
+            return 'business';
+        if (/design|pattern|architecture/.test(context))
+            return 'design';
+        if (/research|study|analysis/.test(context))
+            return 'academic';
+        return 'general';
+    }
+    /**
+     * Detect concept category
+     */
+    detectCategory(conceptName) {
+        if (/pattern|strategy|factory|singleton/.test(conceptName))
+            return 'pattern';
+        if (/principle|rule|law/.test(conceptName))
+            return 'principle';
+        if (/method|approach|technique/.test(conceptName))
+            return 'method';
+        if (/entity|object|model/.test(conceptName))
+            return 'entity';
+        return 'concept';
+    }
+    /**
+     * Detect manifestation form from context
+     */
+    detectManifestationForm(context) {
+        if (context.includes('definition') || context.includes('is defined as'))
+            return 'definition';
+        if (context.includes('example') || context.includes('for instance'))
+            return 'example';
+        if (context.includes('implements') || context.includes('function'))
+            return 'implementation';
+        if (context.includes('discussed') || context.includes('explains'))
+            return 'discussion';
+        return 'usage';
+    }
+    /**
+     * Extract context around a position
+     */
+    extractContext(text, position, radius = 150) {
+        const start = Math.max(0, position - radius);
+        const end = Math.min(text.length, position + radius);
+        return text.slice(start, end);
+    }
+    /**
+     * Clear concept cache
+     */
+    clearCache(conceptId) {
+        if (conceptId) {
+            this.conceptCache.delete(conceptId);
+        }
+        else {
+            this.conceptCache.clear();
+        }
+    }
+}
+//# sourceMappingURL=ConceptSystem.js.map

package/dist/vfs/EntityManager.d.ts

@@ -0,0 +1,75 @@
+/**
+ * EntityManager Base Class
+ *
+ * Provides standardized entity ID management for all Knowledge Layer components
+ * Solves the root cause of ID mismatch issues by establishing clear patterns
+ */
+import { Brainy } from '../brainy.js';
+import { NounType } from '../types/graphTypes.js';
+/**
+ * Standard entity structure used by all Knowledge Layer components
+ */
+export interface ManagedEntity {
+    /** Domain-specific ID (for external references) */
+    id: string;
+    /** The actual Brainy entity ID (for internal operations) */
+    brainyId?: string;
+    /** Entity metadata */
+    [key: string]: any;
+}
+/**
+ * EntityManager Base Class
+ *
+ * All Knowledge Layer components should extend this to get standardized:
+ * - Entity storage and retrieval
+ * - ID management and mapping
+ * - Query patterns
+ * - Relationship creation
+ */
+export declare abstract class EntityManager {
+    protected brain: Brainy;
+    protected systemName: string;
+    private idMappings;
+    private brainyToMappings;
+    constructor(brain: Brainy, systemName: string);
+    /**
+     * Store an entity with proper ID management
+     */
+    protected storeEntity<T extends ManagedEntity>(entity: T, nounType: NounType, embedding?: number[], data?: any): Promise<string>;
+    /**
+     * Update an existing entity
+     */
+    protected updateEntity<T extends ManagedEntity>(entity: T, embedding?: number[]): Promise<void>;
+    /**
+     * Retrieve entity by domain ID
+     */
+    protected getEntity<T extends ManagedEntity>(domainId: string): Promise<T | null>;
+    /**
+     * Find entities by metadata criteria
+     */
+    protected findEntities<T extends ManagedEntity>(criteria: Record<string, any>, nounType?: NounType, limit?: number): Promise<T[]>;
+    /**
+     * Create relationship between entities using proper Brainy IDs
+     */
+    protected createRelationship(fromDomainId: string, toDomainId: string, relationshipType: any, metadata?: Record<string, any>): Promise<void>;
+    /**
+     * Get Brainy ID for a domain ID
+     */
+    protected getBrainyId(domainId: string): Promise<string | null>;
+    /**
+     * Get domain ID for a Brainy ID
+     */
+    protected getDomainId(brainyId: string): string | null;
+    /**
+     * Delete entity by domain ID
+     */
+    protected deleteEntity(domainId: string): Promise<void>;
+    /**
+     * Clear ID mapping cache (useful for tests)
+     */
+    protected clearMappingCache(): void;
+    /**
+     * Batch load mappings for performance
+     */
+    protected loadMappings(domainIds: string[]): Promise<void>;
+}