@soulcraft/brainy 3.49.0 → 3.50.0

package/dist/utils/metadataIndex.js

@@ -10,6 +10,7 @@ import { TypeUtils, NOUN_TYPE_COUNT, VERB_TYPE_COUNT } from '../types/graphTypes
 import { SparseIndex, ChunkManager, AdaptiveChunkingStrategy } from './metadataIndexChunking.js';
 import { EntityIdMapper } from './entityIdMapper.js';
 import { RoaringBitmap32 } from 'roaring-wasm';
+import { FieldTypeInference } from './fieldTypeInference.js';
 export class MetadataIndexManager {
     constructor(storage, config = {}) {
         this.isRebuilding = false;
@@ -81,6 +82,8 @@ export class MetadataIndexManager {
         // Initialize chunking system (v3.42.0) with roaring bitmap support
         this.chunkManager = new ChunkManager(storage, this.idMapper);
         this.chunkingStrategy = new AdaptiveChunkingStrategy();
+        // Initialize Field Type Inference (v3.48.0)
+        this.fieldTypeInference = new FieldTypeInference(storage);
         // Lazy load counts from storage statistics on first access
         this.lazyLoadCounts();
     }
@@ -395,6 +398,8 @@ export class MetadataIndexManager {
                 const data = await this.storage.getMetadata(indexPath);
                 if (data) {
                     const sparseIndex = SparseIndex.fromJSON(data);
+                    // CRITICAL: Initialize chunk ID counter from existing chunks to prevent ID conflicts
+                    this.chunkManager.initializeNextChunkId(field, sparseIndex);
                     // Add to unified cache (sparse indices are expensive to rebuild)
                     const size = JSON.stringify(data).length;
                     this.unifiedCache.set(unifiedKey, sparseIndex, 'metadata', size, 200);
@@ -742,27 +747,54 @@ export class MetadataIndexManager {
             .toLowerCase();
     }
     /**
-     * Normalize value for consistent indexing with smart optimization
+     * Normalize value for consistent indexing with VALUE-BASED temporal detection
+     *
+     * v3.48.0: Replaced unreliable field name pattern matching with production-ready
+     * value-based detection (DuckDB-inspired). Analyzes actual data values, not names.
+     *
+     * NO FALLBACKS - Pure value-based detection only.
      */
     normalizeValue(value, field) {
         if (value === null || value === undefined)
             return '__NULL__';
         if (typeof value === 'boolean')
             return value ? '__TRUE__' : '__FALSE__';
-        // ALWAYS apply bucketing to temporal fields (prevents pollution from the start!)
-        // This is the key fix: don't wait for cardinality stats, just bucket immediately
-        if (field && typeof value === 'number') {
-            const fieldLower = field.toLowerCase();
-            const isTemporal = fieldLower.includes('time') || fieldLower.includes('date') ||
-                fieldLower.includes('accessed') || fieldLower.includes('modified') ||
-                fieldLower.includes('created') || fieldLower.includes('updated');
-            if (isTemporal) {
-                // Apply time bucketing immediately (no need to wait for stats)
-                const bucketSize = this.TIMESTAMP_PRECISION_MS; // 1 minute buckets
+        // VALUE-BASED temporal detection (no pattern matching!)
+        // Analyze the VALUE itself to determine if it's a timestamp
+        if (typeof value === 'number') {
+            // Check if value looks like a Unix timestamp (2000-01-01 to 2100-01-01)
+            const MIN_TIMESTAMP_S = 946684800; // 2000-01-01 in seconds
+            const MAX_TIMESTAMP_S = 4102444800; // 2100-01-01 in seconds
+            const MIN_TIMESTAMP_MS = MIN_TIMESTAMP_S * 1000;
+            const MAX_TIMESTAMP_MS = MAX_TIMESTAMP_S * 1000;
+            const isTimestampSeconds = value >= MIN_TIMESTAMP_S && value <= MAX_TIMESTAMP_S;
+            const isTimestampMilliseconds = value >= MIN_TIMESTAMP_MS && value <= MAX_TIMESTAMP_MS;
+            if (isTimestampSeconds || isTimestampMilliseconds) {
+                // VALUE is a timestamp! Apply 1-minute bucketing
+                const bucketSize = this.TIMESTAMP_PRECISION_MS; // 60000ms = 1 minute
                 const bucketed = Math.floor(value / bucketSize) * bucketSize;
                 return bucketed.toString();
             }
         }
+        // Check if string value is ISO 8601 datetime
+        if (typeof value === 'string') {
+            // ISO 8601 pattern: YYYY-MM-DDTHH:MM:SS...
+            const iso8601Pattern = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/;
+            if (iso8601Pattern.test(value)) {
+                // VALUE is an ISO 8601 datetime! Convert to timestamp and bucket
+                try {
+                    const timestamp = new Date(value).getTime();
+                    if (!isNaN(timestamp)) {
+                        const bucketSize = this.TIMESTAMP_PRECISION_MS;
+                        const bucketed = Math.floor(timestamp / bucketSize) * bucketSize;
+                        return bucketed.toString();
+                    }
+                }
+                catch {
+                    // Not a valid date, treat as string
+                }
+            }
+        }
         // Apply smart normalization based on field statistics (for non-temporal fields)
         if (field && this.fieldStats.has(field)) {
             const stats = this.fieldStats.get(field);

package/dist/utils/metadataIndexChunking.d.ts

@@ -286,6 +286,13 @@ export declare class ChunkManager {
      * Get chunk storage path
      */
     private getChunkPath;
+    /**
+     * Initialize nextChunkId counter from existing sparse index
+     * CRITICAL: Must be called when loading sparse index to prevent ID conflicts
+     * @param field Field name
+     * @param sparseIndex Loaded sparse index containing existing chunk descriptors
+     */
+    initializeNextChunkId(field: string, sparseIndex: SparseIndex): void;
     /**
      * Get next available chunk ID for a field
      */

package/dist/utils/metadataIndexChunking.js

@@ -660,6 +660,20 @@ export class ChunkManager {
     getChunkPath(field, chunkId) {
         return `__chunk__${field}_${chunkId}`;
     }
+    /**
+     * Initialize nextChunkId counter from existing sparse index
+     * CRITICAL: Must be called when loading sparse index to prevent ID conflicts
+     * @param field Field name
+     * @param sparseIndex Loaded sparse index containing existing chunk descriptors
+     */
+    initializeNextChunkId(field, sparseIndex) {
+        const existingChunkIds = sparseIndex.getAllChunkIds();
+        if (existingChunkIds.length > 0) {
+            // Find maximum chunk ID and set next to max + 1
+            const maxChunkId = Math.max(...existingChunkIds);
+            this.nextChunkId.set(field, maxChunkId + 1);
+        }
+    }
     /**
      * Get next available chunk ID for a field
      */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.49.0",
+  "version": "3.50.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",

package/dist/augmentations/KnowledgeAugmentation.d.ts

@@ -1,40 +0,0 @@
-/**
- * Knowledge Layer Augmentation for VFS
- *
- * Adds intelligent features to VFS without modifying core functionality:
- * - Event recording for all operations
- * - Semantic versioning based on content changes
- * - Entity and concept extraction
- * - Git bridge for import/export
- *
- * This is a TRUE augmentation - VFS works perfectly without it
- */
-import { Brainy } from '../brainy.js';
-import { BaseAugmentation } from './brainyAugmentation.js';
-export declare class KnowledgeAugmentation extends BaseAugmentation {
-    name: string;
-    timing: 'after';
-    metadata: 'none';
-    operations: any;
-    priority: number;
-    constructor(config?: any);
-    execute<T = any>(operation: string, params: any, next: () => Promise<T>): Promise<T>;
-    private eventRecorder?;
-    private semanticVersioning?;
-    private entitySystem?;
-    private conceptSystem?;
-    private gitBridge?;
-    private originalMethods;
-    initialize(context: any): Promise<void>;
-    augment(brain: Brainy): Promise<void>;
-    /**
-     * Wrap a VFS method to add Knowledge Layer functionality
-     */
-    private wrapMethod;
-    /**
-     * Add Knowledge Layer methods to VFS
-     */
-    private addKnowledgeMethods;
-    private isSemanticChange;
-    cleanup(brain: Brainy): Promise<void>;
-}

package/dist/augmentations/KnowledgeAugmentation.js

@@ -1,251 +0,0 @@
-/**
- * Knowledge Layer Augmentation for VFS
- *
- * Adds intelligent features to VFS without modifying core functionality:
- * - Event recording for all operations
- * - Semantic versioning based on content changes
- * - Entity and concept extraction
- * - Git bridge for import/export
- *
- * This is a TRUE augmentation - VFS works perfectly without it
- */
-import { BaseAugmentation } from './brainyAugmentation.js';
-import { EventRecorder } from '../vfs/EventRecorder.js';
-import { SemanticVersioning } from '../vfs/SemanticVersioning.js';
-import { PersistentEntitySystem } from '../vfs/PersistentEntitySystem.js';
-import { ConceptSystem } from '../vfs/ConceptSystem.js';
-import { GitBridge } from '../vfs/GitBridge.js';
-export class KnowledgeAugmentation extends BaseAugmentation {
-    constructor(config = {}) {
-        super(config);
-        this.name = 'knowledge';
-        this.timing = 'after'; // Process after VFS operations
-        this.metadata = 'none'; // No metadata access needed
-        this.operations = []; // VFS-specific augmentation, no operation interception
-        this.priority = 100; // Run last
-        this.originalMethods = new Map();
-    }
-    async execute(operation, params, next) {
-        // Pass through - this augmentation works at VFS level, not operation level
-        return await next();
-    }
-    async initialize(context) {
-        await this.augment(context.brain);
-    }
-    async augment(brain) {
-        // Only augment if VFS exists
-        const vfs = brain.vfs?.();
-        if (!vfs) {
-            console.warn('KnowledgeAugmentation: VFS not found, skipping');
-            return;
-        }
-        // Initialize Knowledge Layer components
-        this.eventRecorder = new EventRecorder(brain);
-        this.semanticVersioning = new SemanticVersioning(brain);
-        this.entitySystem = new PersistentEntitySystem(brain);
-        this.conceptSystem = new ConceptSystem(brain);
-        this.gitBridge = new GitBridge(vfs, brain);
-        // Wrap VFS methods to add intelligence WITHOUT slowing them down
-        this.wrapMethod(vfs, 'writeFile', async (original, path, data, options) => {
-            // Call original first (stays fast)
-            const result = await original.call(vfs, path, data, options);
-            // Knowledge processing in background (non-blocking)
-            setImmediate(async () => {
-                try {
-                    // Record event
-                    if (this.eventRecorder) {
-                        await this.eventRecorder.recordEvent({
-                            type: 'write',
-                            path,
-                            content: data,
-                            size: data.length,
-                            author: options?.author || 'system'
-                        });
-                    }
-                    // Check for semantic versioning
-                    if (this.semanticVersioning) {
-                        const existingContent = await vfs.readFile(path).catch(() => null);
-                        const shouldVersion = existingContent && this.isSemanticChange(existingContent, data);
-                        if (shouldVersion) {
-                            await this.semanticVersioning.createVersion(path, data, {
-                                message: 'Automatic semantic version'
-                            });
-                        }
-                    }
-                    // Extract concepts
-                    if (this.conceptSystem && options?.extractConcepts !== false) {
-                        await this.conceptSystem.extractAndLinkConcepts(path, data);
-                    }
-                    // Extract entities
-                    if (this.entitySystem && options?.extractEntities !== false) {
-                        await this.entitySystem.extractEntities(data.toString('utf8'), data);
-                    }
-                }
-                catch (error) {
-                    // Knowledge Layer errors should not affect VFS operations
-                    console.debug('KnowledgeLayer background processing error:', error);
-                }
-            });
-            return result;
-        });
-        this.wrapMethod(vfs, 'unlink', async (original, path) => {
-            const result = await original.call(vfs, path);
-            // Record deletion event
-            setImmediate(async () => {
-                if (this.eventRecorder) {
-                    await this.eventRecorder.recordEvent({
-                        type: 'delete',
-                        path,
-                        author: 'system'
-                    });
-                }
-            });
-            return result;
-        });
-        this.wrapMethod(vfs, 'rename', async (original, oldPath, newPath) => {
-            const result = await original.call(vfs, oldPath, newPath);
-            // Record rename event
-            setImmediate(async () => {
-                if (this.eventRecorder) {
-                    await this.eventRecorder.recordEvent({
-                        type: 'rename',
-                        path: oldPath,
-                        metadata: { newPath },
-                        author: 'system'
-                    });
-                }
-            });
-            return result;
-        });
-        // Add Knowledge Layer methods to VFS
-        this.addKnowledgeMethods(vfs);
-        console.log('✨ Knowledge Layer augmentation enabled');
-    }
-    /**
-     * Wrap a VFS method to add Knowledge Layer functionality
-     */
-    wrapMethod(vfs, methodName, wrapper) {
-        const original = vfs[methodName];
-        if (!original)
-            return;
-        // Store original for cleanup
-        this.originalMethods.set(methodName, original);
-        // Replace with wrapped version
-        vfs[methodName] = async (...args) => {
-            return await wrapper(original, ...args);
-        };
-    }
-    /**
-     * Add Knowledge Layer methods to VFS
-     */
-    addKnowledgeMethods(vfs) {
-        // Event history
-        vfs.getHistory = async (path, options) => {
-            if (!this.eventRecorder)
-                throw new Error('Knowledge Layer not initialized');
-            return await this.eventRecorder.getHistory(path, options);
-        };
-        vfs.reconstructAtTime = async (path, timestamp) => {
-            if (!this.eventRecorder)
-                throw new Error('Knowledge Layer not initialized');
-            return await this.eventRecorder.reconstructFileAtTime(path, timestamp);
-        };
-        // Semantic versioning
-        vfs.getVersions = async (path) => {
-            if (!this.semanticVersioning)
-                throw new Error('Knowledge Layer not initialized');
-            return await this.semanticVersioning.getVersions(path);
-        };
-        vfs.restoreVersion = async (path, versionId) => {
-            if (!this.semanticVersioning)
-                throw new Error('Knowledge Layer not initialized');
-            const version = await this.semanticVersioning.getVersion(path, versionId);
-            if (version) {
-                await vfs.writeFile(path, version);
-            }
-        };
-        // Entities
-        vfs.findEntity = async (query) => {
-            if (!this.entitySystem)
-                throw new Error('Knowledge Layer not initialized');
-            return await this.entitySystem.findEntity(query);
-        };
-        vfs.getEntityAppearances = async (entityId) => {
-            if (!this.entitySystem)
-                throw new Error('Knowledge Layer not initialized');
-            return await this.entitySystem.getEvolution(entityId);
-        };
-        // Concepts
-        vfs.getConcepts = async (path) => {
-            if (!this.conceptSystem)
-                throw new Error('Knowledge Layer not initialized');
-            const concepts = await this.conceptSystem.findConcepts({ manifestedIn: path });
-            return concepts;
-        };
-        vfs.getConceptGraph = async (options) => {
-            if (!this.conceptSystem)
-                throw new Error('Knowledge Layer not initialized');
-            return await this.conceptSystem.getConceptGraph(options);
-        };
-        // Git bridge
-        vfs.exportToGit = async (vfsPath, gitPath) => {
-            if (!this.gitBridge)
-                throw new Error('Knowledge Layer not initialized');
-            return await this.gitBridge.exportToGit(vfsPath, gitPath);
-        };
-        vfs.importFromGit = async (gitPath, vfsPath) => {
-            if (!this.gitBridge)
-                throw new Error('Knowledge Layer not initialized');
-            return await this.gitBridge.importFromGit(gitPath, vfsPath);
-        };
-        // Temporal coupling
-        vfs.findTemporalCoupling = async (path, windowMs) => {
-            if (!this.eventRecorder)
-                throw new Error('Knowledge Layer not initialized');
-            return await this.eventRecorder.findTemporalCoupling(path, windowMs);
-        };
-    }
-    isSemanticChange(oldContent, newContent) {
-        // Simple heuristic - significant size change or different content
-        const oldStr = oldContent.toString('utf8');
-        const newStr = newContent.toString('utf8');
-        // Check for significant size change (>10%)
-        const sizeDiff = Math.abs(oldStr.length - newStr.length) / oldStr.length;
-        if (sizeDiff > 0.1)
-            return true;
-        // Check for structural changes (simplified)
-        const oldLines = oldStr.split('\n').filter(l => l.trim());
-        const newLines = newStr.split('\n').filter(l => l.trim());
-        // Different number of non-empty lines
-        return Math.abs(oldLines.length - newLines.length) > 5;
-    }
-    async cleanup(brain) {
-        const vfs = brain.vfs?.();
-        if (!vfs)
-            return;
-        // Restore original methods
-        for (const [methodName, original] of this.originalMethods) {
-            vfs[methodName] = original;
-        }
-        // Remove added methods
-        delete vfs.getHistory;
-        delete vfs.reconstructAtTime;
-        delete vfs.getVersions;
-        delete vfs.restoreVersion;
-        delete vfs.findEntity;
-        delete vfs.getEntityAppearances;
-        delete vfs.getConcepts;
-        delete vfs.getConceptGraph;
-        delete vfs.exportToGit;
-        delete vfs.importFromGit;
-        delete vfs.findTemporalCoupling;
-        // Clean up components
-        this.eventRecorder = undefined;
-        this.semanticVersioning = undefined;
-        this.entitySystem = undefined;
-        this.conceptSystem = undefined;
-        this.gitBridge = undefined;
-        console.log('Knowledge Layer augmentation removed');
-    }
-}
-//# sourceMappingURL=KnowledgeAugmentation.js.map

package/dist/query/typeInference.d.ts

@@ -1,158 +0,0 @@
-/**
- * Type Inference System - Phase 3: Type-First Query Optimization
- *
- * Automatically infers NounTypes from natural language queries using keyword-based
- * heuristics for fast O(1) type detection.
- *
- * Performance Guarantee: < 1ms per query
- * Accuracy Target: > 80%
- *
- * Examples:
- * - "Find engineers in San Francisco" → [Person, Location]
- * - "Show documents about AI" → [Document, Concept]
- * - "List companies in tech sector" → [Organization, Topic]
- */
-import { NounType } from '../types/graphTypes.js';
-/**
- * Result of type inference with confidence score
- */
-export interface TypeInference {
-    type: NounType;
-    confidence: number;
-    matchedKeywords: string[];
-}
-/**
- * Configuration for type inference behavior
- */
-export interface TypeInferenceConfig {
-    /**
-     * Minimum confidence threshold to include a type (default: 0.4)
-     */
-    minConfidence?: number;
-    /**
-     * Maximum number of types to return (default: 5)
-     */
-    maxTypes?: number;
-    /**
-     * Enable debug logging (default: false)
-     */
-    debug?: boolean;
-    /**
-     * Enable vector similarity fallback for unknown words (default: false)
-     * When enabled, queries with low keyword confidence trigger vector-based type inference
-     */
-    enableVectorFallback?: boolean;
-    /**
-     * Minimum confidence threshold to trigger vector fallback (default: 0.7)
-     * If keyword matching produces confidence below this, vector fallback is used
-     */
-    fallbackConfidenceThreshold?: number;
-    /**
-     * Minimum similarity score for vector-based type matches (default: 0.5)
-     */
-    vectorThreshold?: number;
-}
-/**
- * Type Inference System
- *
- * Uses keyword matching for fast type detection from natural language.
- * Designed for billion-scale performance with minimal latency.
- */
-export declare class TypeInferenceSystem {
-    private keywordMap;
-    private phraseMap;
-    private config;
-    private typeEmbeddings;
-    private embedder;
-    constructor(config?: TypeInferenceConfig);
-    /**
-     * Infer noun types from a natural language query (synchronous keyword matching only)
-     * For hybrid mode with vector fallback, use inferTypesAsync()
-     *
-     * @param query - Natural language query string
-     * @returns Array of type inferences sorted by confidence (highest first)
-     */
-    inferTypes(query: string): TypeInference[];
-    /**
-     * Infer noun types with hybrid approach: keyword matching + optional vector fallback
-     * This is the async version that supports vector similarity fallback
-     *
-     * @param query - Natural language query string
-     * @returns Promise resolving to array of type inferences
-     */
-    inferTypesAsync(query: string): Promise<TypeInference[]>;
-    /**
-     * Internal: Keyword-based type inference (synchronous, fast)
-     */
-    private inferTypesViaKeywords;
-    /**
-     * Internal: Hybrid inference with vector fallback (asynchronous)
-     */
-    private inferTypesWithFallback;
-    /**
-     * Match multi-word phrases in query
-     */
-    private matchPhrases;
-    /**
-     * Match individual keywords in query
-     */
-    private matchKeywords;
-    /**
-     * Find closest keyword using edit distance (for typo correction)
-     * Allows edit distance 1-2 depending on word length
-     */
-    private findFuzzyKeywordMatch;
-    /**
-     * Calculate Levenshtein (edit) distance between two strings
-     */
-    private levenshteinDistance;
-    /**
-     * Update type score with new match
-     */
-    private updateTypeScore;
-    /**
-     * Load pre-compiled type embeddings from embeddedTypeEmbeddings.ts
-     */
-    private loadTypeEmbeddings;
-    /**
-     * Lazy-load TransformerEmbedding model (only when vector fallback is triggered)
-     */
-    private loadEmbedder;
-    /**
-     * Calculate cosine similarity between two vectors
-     */
-    private cosineSimilarity;
-    /**
-     * Infer types using vector similarity against pre-compiled type embeddings
-     */
-    private inferTypesViaVectorSimilarity;
-    /**
-     * Merge keyword-based and vector-based results
-     * Prioritizes keyword results (explicit matches) over vector results (semantic matches)
-     */
-    private mergeResults;
-    /**
-     * Build keyword dictionary for single-word matching
-     */
-    private buildKeywordMap;
-    /**
-     * Build phrase dictionary for multi-word matching
-     */
-    private buildPhraseMap;
-    /**
-     * Get statistics about the inference system
-     */
-    getStats(): {
-        keywordCount: number;
-        phraseCount: number;
-        config: Required<TypeInferenceConfig>;
-    };
-}
-/**
- * Get or create the global TypeInferenceSystem instance
- */
-export declare function getTypeInferenceSystem(config?: TypeInferenceConfig): TypeInferenceSystem;
-/**
- * Convenience function to infer types from a query
- */
-export declare function inferTypes(query: string, config?: TypeInferenceConfig): TypeInference[];