@soulcraft/brainy 6.2.0 → 6.2.1

package/dist/augmentations/KnowledgeAugmentation.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Knowledge Layer Augmentation for VFS
+ *
+ * Adds intelligent features to VFS without modifying core functionality:
+ * - Event recording for all operations
+ * - Semantic versioning based on content changes
+ * - Entity and concept extraction
+ * - Git bridge for import/export
+ *
+ * This is a TRUE augmentation - VFS works perfectly without it
+ */
+import { Brainy } from '../brainy.js';
+import { BaseAugmentation } from './brainyAugmentation.js';
+export declare class KnowledgeAugmentation extends BaseAugmentation {
+    name: string;
+    timing: 'after';
+    metadata: 'none';
+    operations: any;
+    priority: number;
+    constructor(config?: any);
+    execute<T = any>(operation: string, params: any, next: () => Promise<T>): Promise<T>;
+    private eventRecorder?;
+    private semanticVersioning?;
+    private entitySystem?;
+    private conceptSystem?;
+    private gitBridge?;
+    private originalMethods;
+    initialize(context: any): Promise<void>;
+    augment(brain: Brainy): Promise<void>;
+    /**
+     * Wrap a VFS method to add Knowledge Layer functionality
+     */
+    private wrapMethod;
+    /**
+     * Add Knowledge Layer methods to VFS
+     */
+    private addKnowledgeMethods;
+    private isSemanticChange;
+    cleanup(brain: Brainy): Promise<void>;
+}

package/dist/augmentations/KnowledgeAugmentation.js

@@ -0,0 +1,251 @@
+/**
+ * Knowledge Layer Augmentation for VFS
+ *
+ * Adds intelligent features to VFS without modifying core functionality:
+ * - Event recording for all operations
+ * - Semantic versioning based on content changes
+ * - Entity and concept extraction
+ * - Git bridge for import/export
+ *
+ * This is a TRUE augmentation - VFS works perfectly without it
+ */
+import { BaseAugmentation } from './brainyAugmentation.js';
+import { EventRecorder } from '../vfs/EventRecorder.js';
+import { SemanticVersioning } from '../vfs/SemanticVersioning.js';
+import { PersistentEntitySystem } from '../vfs/PersistentEntitySystem.js';
+import { ConceptSystem } from '../vfs/ConceptSystem.js';
+import { GitBridge } from '../vfs/GitBridge.js';
+export class KnowledgeAugmentation extends BaseAugmentation {
+    constructor(config = {}) {
+        super(config);
+        this.name = 'knowledge';
+        this.timing = 'after'; // Process after VFS operations
+        this.metadata = 'none'; // No metadata access needed
+        this.operations = []; // VFS-specific augmentation, no operation interception
+        this.priority = 100; // Run last
+        this.originalMethods = new Map();
+    }
+    async execute(operation, params, next) {
+        // Pass through - this augmentation works at VFS level, not operation level
+        return await next();
+    }
+    async initialize(context) {
+        await this.augment(context.brain);
+    }
+    async augment(brain) {
+        // Only augment if VFS exists
+        const vfs = brain.vfs?.();
+        if (!vfs) {
+            console.warn('KnowledgeAugmentation: VFS not found, skipping');
+            return;
+        }
+        // Initialize Knowledge Layer components
+        this.eventRecorder = new EventRecorder(brain);
+        this.semanticVersioning = new SemanticVersioning(brain);
+        this.entitySystem = new PersistentEntitySystem(brain);
+        this.conceptSystem = new ConceptSystem(brain);
+        this.gitBridge = new GitBridge(vfs, brain);
+        // Wrap VFS methods to add intelligence WITHOUT slowing them down
+        this.wrapMethod(vfs, 'writeFile', async (original, path, data, options) => {
+            // Call original first (stays fast)
+            const result = await original.call(vfs, path, data, options);
+            // Knowledge processing in background (non-blocking)
+            setImmediate(async () => {
+                try {
+                    // Record event
+                    if (this.eventRecorder) {
+                        await this.eventRecorder.recordEvent({
+                            type: 'write',
+                            path,
+                            content: data,
+                            size: data.length,
+                            author: options?.author || 'system'
+                        });
+                    }
+                    // Check for semantic versioning
+                    if (this.semanticVersioning) {
+                        const existingContent = await vfs.readFile(path).catch(() => null);
+                        const shouldVersion = existingContent && this.isSemanticChange(existingContent, data);
+                        if (shouldVersion) {
+                            await this.semanticVersioning.createVersion(path, data, {
+                                message: 'Automatic semantic version'
+                            });
+                        }
+                    }
+                    // Extract concepts
+                    if (this.conceptSystem && options?.extractConcepts !== false) {
+                        await this.conceptSystem.extractAndLinkConcepts(path, data);
+                    }
+                    // Extract entities
+                    if (this.entitySystem && options?.extractEntities !== false) {
+                        await this.entitySystem.extractEntities(data.toString('utf8'), data);
+                    }
+                }
+                catch (error) {
+                    // Knowledge Layer errors should not affect VFS operations
+                    console.debug('KnowledgeLayer background processing error:', error);
+                }
+            });
+            return result;
+        });
+        this.wrapMethod(vfs, 'unlink', async (original, path) => {
+            const result = await original.call(vfs, path);
+            // Record deletion event
+            setImmediate(async () => {
+                if (this.eventRecorder) {
+                    await this.eventRecorder.recordEvent({
+                        type: 'delete',
+                        path,
+                        author: 'system'
+                    });
+                }
+            });
+            return result;
+        });
+        this.wrapMethod(vfs, 'rename', async (original, oldPath, newPath) => {
+            const result = await original.call(vfs, oldPath, newPath);
+            // Record rename event
+            setImmediate(async () => {
+                if (this.eventRecorder) {
+                    await this.eventRecorder.recordEvent({
+                        type: 'rename',
+                        path: oldPath,
+                        metadata: { newPath },
+                        author: 'system'
+                    });
+                }
+            });
+            return result;
+        });
+        // Add Knowledge Layer methods to VFS
+        this.addKnowledgeMethods(vfs);
+        console.log('✨ Knowledge Layer augmentation enabled');
+    }
+    /**
+     * Wrap a VFS method to add Knowledge Layer functionality
+     */
+    wrapMethod(vfs, methodName, wrapper) {
+        const original = vfs[methodName];
+        if (!original)
+            return;
+        // Store original for cleanup
+        this.originalMethods.set(methodName, original);
+        // Replace with wrapped version
+        vfs[methodName] = async (...args) => {
+            return await wrapper(original, ...args);
+        };
+    }
+    /**
+     * Add Knowledge Layer methods to VFS
+     */
+    addKnowledgeMethods(vfs) {
+        // Event history
+        vfs.getHistory = async (path, options) => {
+            if (!this.eventRecorder)
+                throw new Error('Knowledge Layer not initialized');
+            return await this.eventRecorder.getHistory(path, options);
+        };
+        vfs.reconstructAtTime = async (path, timestamp) => {
+            if (!this.eventRecorder)
+                throw new Error('Knowledge Layer not initialized');
+            return await this.eventRecorder.reconstructFileAtTime(path, timestamp);
+        };
+        // Semantic versioning
+        vfs.getVersions = async (path) => {
+            if (!this.semanticVersioning)
+                throw new Error('Knowledge Layer not initialized');
+            return await this.semanticVersioning.getVersions(path);
+        };
+        vfs.restoreVersion = async (path, versionId) => {
+            if (!this.semanticVersioning)
+                throw new Error('Knowledge Layer not initialized');
+            const version = await this.semanticVersioning.getVersion(path, versionId);
+            if (version) {
+                await vfs.writeFile(path, version);
+            }
+        };
+        // Entities
+        vfs.findEntity = async (query) => {
+            if (!this.entitySystem)
+                throw new Error('Knowledge Layer not initialized');
+            return await this.entitySystem.findEntity(query);
+        };
+        vfs.getEntityAppearances = async (entityId) => {
+            if (!this.entitySystem)
+                throw new Error('Knowledge Layer not initialized');
+            return await this.entitySystem.getEvolution(entityId);
+        };
+        // Concepts
+        vfs.getConcepts = async (path) => {
+            if (!this.conceptSystem)
+                throw new Error('Knowledge Layer not initialized');
+            const concepts = await this.conceptSystem.findConcepts({ manifestedIn: path });
+            return concepts;
+        };
+        vfs.getConceptGraph = async (options) => {
+            if (!this.conceptSystem)
+                throw new Error('Knowledge Layer not initialized');
+            return await this.conceptSystem.getConceptGraph(options);
+        };
+        // Git bridge
+        vfs.exportToGit = async (vfsPath, gitPath) => {
+            if (!this.gitBridge)
+                throw new Error('Knowledge Layer not initialized');
+            return await this.gitBridge.exportToGit(vfsPath, gitPath);
+        };
+        vfs.importFromGit = async (gitPath, vfsPath) => {
+            if (!this.gitBridge)
+                throw new Error('Knowledge Layer not initialized');
+            return await this.gitBridge.importFromGit(gitPath, vfsPath);
+        };
+        // Temporal coupling
+        vfs.findTemporalCoupling = async (path, windowMs) => {
+            if (!this.eventRecorder)
+                throw new Error('Knowledge Layer not initialized');
+            return await this.eventRecorder.findTemporalCoupling(path, windowMs);
+        };
+    }
+    isSemanticChange(oldContent, newContent) {
+        // Simple heuristic - significant size change or different content
+        const oldStr = oldContent.toString('utf8');
+        const newStr = newContent.toString('utf8');
+        // Check for significant size change (>10%)
+        const sizeDiff = Math.abs(oldStr.length - newStr.length) / oldStr.length;
+        if (sizeDiff > 0.1)
+            return true;
+        // Check for structural changes (simplified)
+        const oldLines = oldStr.split('\n').filter(l => l.trim());
+        const newLines = newStr.split('\n').filter(l => l.trim());
+        // Different number of non-empty lines
+        return Math.abs(oldLines.length - newLines.length) > 5;
+    }
+    async cleanup(brain) {
+        const vfs = brain.vfs?.();
+        if (!vfs)
+            return;
+        // Restore original methods
+        for (const [methodName, original] of this.originalMethods) {
+            vfs[methodName] = original;
+        }
+        // Remove added methods
+        delete vfs.getHistory;
+        delete vfs.reconstructAtTime;
+        delete vfs.getVersions;
+        delete vfs.restoreVersion;
+        delete vfs.findEntity;
+        delete vfs.getEntityAppearances;
+        delete vfs.getConcepts;
+        delete vfs.getConceptGraph;
+        delete vfs.exportToGit;
+        delete vfs.importFromGit;
+        delete vfs.findTemporalCoupling;
+        // Clean up components
+        this.eventRecorder = undefined;
+        this.semanticVersioning = undefined;
+        this.entitySystem = undefined;
+        this.conceptSystem = undefined;
+        this.gitBridge = undefined;
+        console.log('Knowledge Layer augmentation removed');
+    }
+}
+//# sourceMappingURL=KnowledgeAugmentation.js.map

package/dist/brainy.d.ts

@@ -1566,9 +1566,11 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
     get counts(): {
         entities: () => number;
         relationships: () => number;
-        byType: (type?: string) => number | {
-            [k: string]: number;
-        };
+        byType: (typeOrOptions?: string | {
+            excludeVFS?: boolean;
+        }, options?: {
+            excludeVFS?: boolean;
+        }) => Promise<number | Record<string, number>>;
         byTypeEnum: (type: NounType) => number;
         topTypes: (n?: number) => NounType[];
         topVerbTypes: (n?: number) => VerbType[];
@@ -1579,12 +1581,12 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
         };
         byCriteria: (field: string, value: any) => Promise<number>;
         getAllTypeCounts: () => Map<string, number>;
-        getStats: () => {
+        getStats: (options?: {
+            excludeVFS?: boolean;
+        }) => Promise<{
             entities: {
                 total: number;
-                byType: {
-                    [k: string]: number;
-                };
+                byType: Record<string, number>;
             };
             relationships: {
                 totalRelationships: number;
@@ -1594,7 +1596,7 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
                 totalNodes: number;
             };
             density: number;
-        };
+        }>;
     };
     /**
      * Augmentations API - Clean and simple
@@ -1607,14 +1609,16 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
     /**
      * Get complete statistics - convenience method
      * For more granular counting, use brain.counts API
+     * v6.2.1: Added optional excludeVFS using Roaring bitmap intersection
+     * @param options Optional settings - excludeVFS: filter out VFS entities
      * @returns Complete statistics including entities, relationships, and density
      */
-    getStats(): {
+    getStats(options?: {
+        excludeVFS?: boolean;
+    }): Promise<{
         entities: {
             total: number;
-            byType: {
-                [k: string]: number;
-            };
+            byType: Record<string, number>;
         };
         relationships: {
             totalRelationships: number;
@@ -1624,7 +1628,7 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
             totalNodes: number;
         };
         density: number;
-    };
+    }>;
     /**
      * Parse natural language query using advanced NLP with 220+ patterns
      * The embedding model is always available as it's core to Brainy's functionality

package/dist/brainy.js

@@ -768,7 +768,13 @@ export class Brainy {
                     }
                 }
                 // Operation 5-6: Update metadata index (remove old, add new)
-                tx.addOperation(new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, existing.metadata));
+                // v6.2.1: Fix - Include type in removal metadata so noun index is properly updated
+                // existing.metadata only contains custom fields, not 'type' which maps to 'noun'
+                const removalMetadata = {
+                    ...existing.metadata,
+                    type: existing.type // Include type so it maps to 'noun' during removal
+                };
+                tx.addOperation(new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata));
                 tx.addOperation(new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing));
             });
         });
@@ -3405,7 +3411,39 @@ export class Brainy {
             // O(1) total relationship count
             relationships: () => this.graphIndex.getTotalRelationshipCount(),
             // O(1) count by type (string-based, backward compatible)
-            byType: (type) => {
+            // v6.2.1: Added optional excludeVFS using Roaring bitmap intersection
+            byType: async (typeOrOptions, options) => {
+                // Handle overloaded signature: byType(type), byType({ excludeVFS }), byType(type, { excludeVFS })
+                let type;
+                let excludeVFS = false;
+                if (typeof typeOrOptions === 'string') {
+                    type = typeOrOptions;
+                    excludeVFS = options?.excludeVFS ?? false;
+                }
+                else if (typeOrOptions && typeof typeOrOptions === 'object') {
+                    excludeVFS = typeOrOptions.excludeVFS ?? false;
+                }
+                if (excludeVFS) {
+                    const allCounts = this.metadataIndex.getAllEntityCounts();
+                    // Uses Roaring bitmap intersection - hardware accelerated
+                    const vfsCounts = await this.metadataIndex.getAllVFSEntityCounts();
+                    if (type) {
+                        const total = allCounts.get(type) || 0;
+                        const vfs = vfsCounts.get(type) || 0;
+                        return total - vfs;
+                    }
+                    // Return all counts with VFS subtracted
+                    const result = {};
+                    for (const [t, total] of allCounts) {
+                        const vfs = vfsCounts.get(t) || 0;
+                        const nonVfs = total - vfs;
+                        if (nonVfs > 0) {
+                            result[t] = nonVfs;
+                        }
+                    }
+                    return result;
+                }
+                // Default path (unchanged) - synchronous for backward compatibility
                 if (type) {
                     return this.metadataIndex.getEntityCountByType(type);
                 }
@@ -3447,7 +3485,32 @@ export class Brainy {
             // Get all type counts as Map for performance-critical operations
             getAllTypeCounts: () => this.metadataIndex.getAllEntityCounts(),
             // Get complete statistics
-            getStats: () => {
+            // v6.2.1: Added optional excludeVFS using Roaring bitmap intersection
+            getStats: async (options) => {
+                if (options?.excludeVFS) {
+                    const allCounts = this.metadataIndex.getAllEntityCounts();
+                    // Uses Roaring bitmap intersection - hardware accelerated
+                    const vfsCounts = await this.metadataIndex.getAllVFSEntityCounts();
+                    // Compute non-VFS counts via subtraction
+                    const byType = {};
+                    let total = 0;
+                    for (const [type, count] of allCounts) {
+                        const vfs = vfsCounts.get(type) || 0;
+                        const nonVfs = count - vfs;
+                        if (nonVfs > 0) {
+                            byType[type] = nonVfs;
+                            total += nonVfs;
+                        }
+                    }
+                    const entityStats = { total, byType };
+                    const relationshipStats = this.graphIndex.getRelationshipStats();
+                    return {
+                        entities: entityStats,
+                        relationships: relationshipStats,
+                        density: total > 0 ? relationshipStats.totalRelationships / total : 0
+                    };
+                }
+                // Default path (unchanged) - synchronous for backward compatibility
                 const entityStats = {
                     total: this.metadataIndex.getTotalEntityCount(),
                     byType: Object.fromEntries(this.metadataIndex.getAllEntityCounts())
@@ -3474,10 +3537,12 @@ export class Brainy {
     /**
      * Get complete statistics - convenience method
      * For more granular counting, use brain.counts API
+     * v6.2.1: Added optional excludeVFS using Roaring bitmap intersection
+     * @param options Optional settings - excludeVFS: filter out VFS entities
      * @returns Complete statistics including entities, relationships, and density
      */
-    getStats() {
-        return this.counts.getStats();
+    async getStats(options) {
+        return this.counts.getStats(options);
     }
     // ============= HELPER METHODS =============
     /**

package/dist/importManager.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Import Manager - Comprehensive data import with intelligent type detection
+ *
+ * Handles multiple data sources:
+ * - Direct data (objects, arrays)
+ * - Files (JSON, CSV, text)
+ * - URLs (fetch and parse)
+ * - Streams (for large files)
+ *
+ * Uses NeuralImportAugmentation for intelligent processing
+ */
+import { NounType } from './types/graphTypes.js';
+export interface ImportOptions {
+    source?: 'data' | 'file' | 'url' | 'auto';
+    format?: 'json' | 'csv' | 'text' | 'yaml' | 'auto';
+    batchSize?: number;
+    autoDetect?: boolean;
+    typeHint?: NounType;
+    extractRelationships?: boolean;
+    csvDelimiter?: string;
+    csvHeaders?: boolean;
+    parallel?: boolean;
+    maxConcurrency?: number;
+}
+export interface ImportResult {
+    success: boolean;
+    nouns: string[];
+    verbs: string[];
+    errors: string[];
+    stats: {
+        total: number;
+        imported: number;
+        failed: number;
+        relationships: number;
+    };
+}
+export declare class ImportManager {
+    private neuralImport;
+    private typeMatcher;
+    private brain;
+    constructor(brain: any);
+    /**
+     * Initialize the import manager
+     */
+    init(): Promise<void>;
+    /**
+     * Main import method - handles all sources
+     */
+    import(source: string | Buffer | any[] | any, options?: ImportOptions): Promise<ImportResult>;
+    /**
+     * Import from file
+     */
+    importFile(filePath: string, options?: ImportOptions): Promise<ImportResult>;
+    /**
+     * Import from URL
+     */
+    importUrl(url: string, options?: ImportOptions): Promise<ImportResult>;
+    /**
+     * Detect source type
+     */
+    private detectSourceType;
+    /**
+     * Detect format from file path
+     */
+    private detectFormatFromPath;
+    /**
+     * Read file
+     */
+    private readFile;
+    /**
+     * Fetch from URL
+     */
+    private fetchFromUrl;
+}
+/**
+ * Create an import manager instance
+ */
+export declare function createImportManager(brain: any): ImportManager;