@soulcraft/brainy 6.6.0 → 6.6.2

package/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [6.6.2](https://github.com/soulcraftlabs/brainy/compare/v6.6.1...v6.6.2) (2026-01-05)
+
+- fix: resolve update() v5.11.1 regression + skip flaky tests for release (106f654)
+- fix(metadata-index): delete chunk files during rebuild to prevent 77x overcounting (386666d)
+
+
 ## [6.4.0](https://github.com/soulcraftlabs/brainy/compare/v6.3.2...v6.4.0) (2025-12-11)
 
 ### ⚡ Performance

package/dist/brainy.js

@@ -80,7 +80,6 @@ export class Brainy {
                 ...this.config,
                 ...configOverrides,
                 storage: { ...this.config.storage, ...configOverrides.storage },
-                model: { ...this.config.model, ...configOverrides.model },
                 index: { ...this.config.index, ...configOverrides.index },
                 augmentations: { ...this.config.augmentations, ...configOverrides.augmentations },
                 verbose: configOverrides.verbose ?? this.config.verbose,
@@ -694,8 +693,11 @@ export class Brainy {
         const { validateUpdateParams } = await import('./utils/paramValidation.js');
         validateUpdateParams(params);
         return this.augmentationRegistry.execute('update', params, async () => {
-            // Get existing entity
-            const existing = await this.get(params.id);
+            // Get existing entity with vectors (v6.7.0: fix for v5.11.1 regression)
+            // We need includeVectors: true because:
+            // 1. SaveNounOperation requires the vector
+            // 2. HNSW reindexing operations need the original vector
+            const existing = await this.get(params.id, { includeVectors: true });
             if (!existing) {
                 throw new Error(`Entity ${params.id} not found`);
             }
@@ -4146,10 +4148,6 @@ export class Brainy {
             // No longer throw errors for mismatches - storageFactory now handles this intelligently
             // Both 'gcs' and 'gcs-native' can now use either gcsStorage or gcsNativeStorage
         }
-        // Validate model configuration
-        if (config?.model?.type && !['fast', 'accurate', 'custom'].includes(config.model.type)) {
-            throw new Error(`Invalid model type: ${config.model.type}. Must be one of: fast, accurate, custom`);
-        }
         // Validate numeric configurations
         if (config?.index?.m && (config.index.m < 1 || config.index.m > 128)) {
             throw new Error(`Invalid index m parameter: ${config.index.m}. Must be between 1 and 128`);
@@ -4164,7 +4162,6 @@ export class Brainy {
         const distributedConfig = this.autoDetectDistributed(config?.distributed);
         return {
             storage: config?.storage || { type: 'auto' },
-            model: config?.model || { type: 'fast' },
             index: config?.index || {},
             cache: config?.cache ?? true,
             augmentations: config?.augmentations || {},

package/dist/config/index.d.ts

@@ -2,9 +2,7 @@
  * Zero-Configuration System
  * Main entry point for all auto-configuration features
  */
-export { autoSelectModelPrecision, ModelPrecision as ModelPrecisionType, // Avoid conflict
-ModelPreset, shouldAutoDownloadModels, getModelPath, logModelConfig } from './modelAutoConfig.js';
-export declare const getModelPrecision: () => "q8";
+export { getModelPrecision, shouldAutoDownloadModels, getModelPath } from './modelAutoConfig.js';
 export { autoDetectStorage, StorageType, StoragePreset, StorageConfigResult, logStorageConfig, type StorageTypeString, type StoragePresetString } from './storageAutoConfig.js';
 export { SharedConfig, SharedConfigManager } from './sharedConfigManager.js';
 export { BrainyZeroConfig, processZeroConfig, createEmbeddingFunctionWithPrecision } from './zeroConfig.js';

package/dist/config/index.js

@@ -2,10 +2,8 @@
  * Zero-Configuration System
  * Main entry point for all auto-configuration features
  */
-// Model configuration
-export { autoSelectModelPrecision, shouldAutoDownloadModels, getModelPath, logModelConfig } from './modelAutoConfig.js';
-// Model precision - Always Q8 now (99% accuracy, 75% smaller)
-export const getModelPrecision = () => 'q8';
+// Model configuration (simplified - always Q8 WASM)
+export { getModelPrecision, shouldAutoDownloadModels, getModelPath } from './modelAutoConfig.js';
 // Storage configuration
 export { autoDetectStorage, StorageType, StoragePreset, logStorageConfig } from './storageAutoConfig.js';
 // Shared configuration for multi-instance

package/dist/config/modelAutoConfig.d.ts

@@ -1,32 +1,25 @@
 /**
- * Model Configuration Auto-Selection
- * Always uses Q8 for optimal size/performance balance (99% accuracy, 75% smaller)
+ * Model Configuration
+ * Brainy uses Q8 WASM embeddings - no configuration needed (zero-config)
  */
-export type ModelPrecision = 'q8';
-export type ModelPreset = 'small' | 'auto';
 interface ModelConfigResult {
-    precision: ModelPrecision;
+    precision: 'q8';
     reason: string;
     autoSelected: boolean;
 }
 /**
- * Auto-select model precision - Always returns Q8
- * Q8 provides 99% accuracy with 75% smaller size
- * @param override - For backward compatibility, ignored
+ * Get model precision configuration
+ * Always returns Q8 - the optimal balance of size and accuracy
  */
-export declare function autoSelectModelPrecision(override?: ModelPrecision | ModelPreset): ModelConfigResult;
+export declare function getModelPrecision(): ModelConfigResult;
 /**
- * Convenience function to check if models need to be downloaded
- * This replaces the need for BRAINY_ALLOW_REMOTE_MODELS
+ * Check if models need to be downloaded
+ * With bundled WASM model, this is rarely needed
  */
 export declare function shouldAutoDownloadModels(): boolean;
 /**
- * Get the model path with intelligent defaults
- * This replaces the need for BRAINY_MODELS_PATH env var
+ * Get the model path
+ * With bundled WASM model, this points to the package assets
  */
 export declare function getModelPath(): string;
-/**
- * Log model configuration decision (only in verbose mode)
- */
-export declare function logModelConfig(config: ModelConfigResult, verbose?: boolean): void;
 export {};

package/dist/config/modelAutoConfig.js

@@ -1,35 +1,16 @@
 /**
- * Model Configuration Auto-Selection
- * Always uses Q8 for optimal size/performance balance (99% accuracy, 75% smaller)
+ * Model Configuration
+ * Brainy uses Q8 WASM embeddings - no configuration needed (zero-config)
  */
 import { isBrowser, isNode } from '../utils/environment.js';
 /**
- * Auto-select model precision - Always returns Q8
- * Q8 provides 99% accuracy with 75% smaller size
- * @param override - For backward compatibility, ignored
+ * Get model precision configuration
+ * Always returns Q8 - the optimal balance of size and accuracy
  */
-export function autoSelectModelPrecision(override) {
-    // Always use Q8 regardless of override for simplicity
-    // Q8 is optimal: 33MB vs 130MB, 99% accuracy retained
-    // Log deprecation notice if FP32 was requested
-    if (typeof override === 'string' && override.toLowerCase().includes('fp32')) {
-        console.log('Note: FP32 precision is deprecated. Using Q8 (99% accuracy, 75% smaller).');
-    }
-    return {
-        precision: 'q8',
-        reason: 'Q8 precision (99% accuracy, 75% smaller)',
-        autoSelected: true
-    };
-}
-/**
- * Automatically detect the best model precision for the environment
- * DEPRECATED: Always returns Q8 now
- */
-function autoDetectBestPrecision() {
-    // Always return Q8 - deprecated function kept for backward compatibility
+export function getModelPrecision() {
     return {
         precision: 'q8',
-        reason: 'Q8 precision (99% accuracy, 75% smaller)',
+        reason: 'Q8 WASM (23MB bundled, no downloads)',
         autoSelected: true
     };
 }
@@ -48,68 +29,25 @@ function isServerlessEnvironment() {
     );
 }
 /**
- * Get available memory in MB
- */
-function getAvailableMemoryMB() {
-    if (isBrowser()) {
-        // @ts-ignore - navigator.deviceMemory is experimental
-        if (navigator.deviceMemory) {
-            // @ts-ignore
-            return navigator.deviceMemory * 1024; // Device memory in GB
-        }
-        return 256; // Conservative default for browsers
-    }
-    if (isNode()) {
-        try {
-            // Try to get memory info synchronously for Node.js
-            // This will be available in Node.js environments
-            if (typeof process !== 'undefined' && process.memoryUsage) {
-                // Use RSS (Resident Set Size) as a proxy for available memory
-                const rss = process.memoryUsage().rss;
-                // Assume we can use up to 4GB or 50% more than current usage
-                return Math.min(4096, Math.floor(rss / (1024 * 1024) * 1.5));
-            }
-        }
-        catch {
-            // Fall through to default
-        }
-        return 1024; // Default 1GB for Node.js
-    }
-    return 512; // Conservative default
-}
-/**
- * Convenience function to check if models need to be downloaded
- * This replaces the need for BRAINY_ALLOW_REMOTE_MODELS
+ * Check if models need to be downloaded
+ * With bundled WASM model, this is rarely needed
  */
 export function shouldAutoDownloadModels() {
-    // Always allow downloads unless explicitly disabled
-    // This eliminates the need for BRAINY_ALLOW_REMOTE_MODELS
+    // Model is bundled - no downloads needed in normal operation
+    // This flag exists for edge cases only
     const explicitlyDisabled = process.env.BRAINY_ALLOW_REMOTE_MODELS === 'false';
-    if (explicitlyDisabled) {
-        console.warn('Model downloads disabled via BRAINY_ALLOW_REMOTE_MODELS=false');
-        return false;
-    }
-    // In production, always allow downloads for seamless operation
-    if (process.env.NODE_ENV === 'production') {
-        return true;
-    }
-    // In development, allow downloads with a one-time notice
-    if (process.env.NODE_ENV === 'development') {
-        return true;
-    }
-    // Default: allow downloads
-    return true;
+    return !explicitlyDisabled;
 }
 /**
- * Get the model path with intelligent defaults
- * This replaces the need for BRAINY_MODELS_PATH env var
+ * Get the model path
+ * With bundled WASM model, this points to the package assets
  */
 export function getModelPath() {
-    // Check if user explicitly set a path (keeping this for advanced users)
+    // Check if user explicitly set a path (for advanced users)
     if (process.env.BRAINY_MODELS_PATH) {
         return process.env.BRAINY_MODELS_PATH;
     }
-    // Browser - use cache API or IndexedDB (handled by transformers.js)
+    // Browser - use cache API or IndexedDB
     if (isBrowser()) {
         return 'browser-cache';
     }
@@ -119,21 +57,10 @@ export function getModelPath() {
     }
     // Node.js - use home directory for persistent storage
     if (isNode()) {
-        // Use process.env.HOME as a fallback
         const homeDir = process.env.HOME || process.env.USERPROFILE || '~';
         return `${homeDir}/.brainy/models`;
     }
     // Fallback
     return './.brainy/models';
 }
-/**
- * Log model configuration decision (only in verbose mode)
- */
-export function logModelConfig(config, verbose = false) {
-    if (!verbose && process.env.NODE_ENV === 'production') {
-        return; // Silent in production unless verbose
-    }
-    const icon = config.autoSelected ? '🤖' : '👤';
-    console.log(`${icon} Model: ${config.precision.toUpperCase()} - ${config.reason}`);
-}
 //# sourceMappingURL=modelAutoConfig.js.map

package/dist/config/sharedConfigManager.d.ts

@@ -2,10 +2,9 @@
  * Shared Configuration Manager
  * Ensures configuration consistency across multiple instances using shared storage
  */
-import { ModelPrecision } from './modelAutoConfig.js';
 export interface SharedConfig {
     version: string;
-    precision: ModelPrecision;
+    precision: 'q8';
     dimensions: number;
     hnswM: number;
     hnswEfConstruction: number;

package/dist/config/zeroConfig.d.ts

@@ -2,7 +2,6 @@
  * Zero-Configuration System for Brainy
  * Provides intelligent defaults while preserving full control
  */
-import { ModelPrecision, ModelPreset } from './modelAutoConfig.js';
 import { StorageType, StoragePreset } from './storageAutoConfig.js';
 /**
  * Simplified configuration interface
@@ -19,15 +18,6 @@ export interface BrainyZeroConfig {
      * - 'reader': Read-only instance for distributed setups (no write operations)
      */
     mode?: 'production' | 'development' | 'minimal' | 'zero' | 'writer' | 'reader';
-    /**
-     * Model precision configuration
-     * - 'fp32': Full precision (best quality, larger size)
-     * - 'q8': Quantized 8-bit (smaller size, slightly lower quality)
-     * - 'fast': Alias for fp32
-     * - 'small': Alias for q8
-     * - 'auto': Auto-detect based on environment (default)
-     */
-    model?: ModelPrecision | ModelPreset;
     /**
      * Storage configuration
      * - 'memory': In-memory only (no persistence)
@@ -62,7 +52,6 @@ export interface BrainyZeroConfig {
  */
 export declare function processZeroConfig(input?: string | BrainyZeroConfig): Promise<any>;
 /**
- * Create embedding function with specified precision
- * This ensures the model precision is respected
+ * Create embedding function (always Q8 WASM)
  */
-export declare function createEmbeddingFunctionWithPrecision(precision: ModelPrecision): Promise<any>;
+export declare function createEmbeddingFunctionWithPrecision(): Promise<any>;

package/dist/config/zeroConfig.js

@@ -2,7 +2,7 @@
  * Zero-Configuration System for Brainy
  * Provides intelligent defaults while preserving full control
  */
-import { autoSelectModelPrecision, getModelPath, shouldAutoDownloadModels } from './modelAutoConfig.js';
+import { getModelPrecision, getModelPath, shouldAutoDownloadModels } from './modelAutoConfig.js';
 import { autoDetectStorage } from './storageAutoConfig.js';
 import { AutoConfiguration } from '../utils/autoConfiguration.js';
 /**
@@ -11,31 +11,26 @@ import { AutoConfiguration } from '../utils/autoConfiguration.js';
 const PRESETS = {
     production: {
         storage: 'disk',
-        model: 'auto',
         features: 'default',
         verbose: false
     },
     development: {
         storage: 'memory',
-        model: 'q8', // Q8 is now the default for all presets
         features: 'full',
         verbose: true
     },
     minimal: {
         storage: 'memory',
-        model: 'q8',
         features: 'minimal',
         verbose: false
     },
     zero: {
         storage: 'auto',
-        model: 'auto',
         features: 'default',
         verbose: false
     },
     writer: {
         storage: 'auto',
-        model: 'auto',
         features: 'minimal',
         verbose: false,
         // Writer-specific settings
@@ -46,7 +41,6 @@ const PRESETS = {
     },
     reader: {
         storage: 'auto',
-        model: 'auto',
         features: 'default',
         verbose: false,
         // Reader-specific settings
@@ -117,7 +111,6 @@ export async function processZeroConfig(input) {
             ...preset,
             ...config,
             // Preserve explicit overrides
-            model: config.model ?? preset.model,
             storage: config.storage ?? preset.storage,
             features: config.features ?? preset.features,
             verbose: config.verbose ?? preset.verbose
@@ -125,8 +118,8 @@ export async function processZeroConfig(input) {
     }
     // Auto-detect environment if not in preset mode
     const environment = detectEnvironmentMode();
-    // Process model configuration
-    const modelConfig = autoSelectModelPrecision(config.model);
+    // Get model configuration (always Q8 WASM)
+    const modelConfig = getModelPrecision();
     // Process storage configuration
     const storageConfig = await autoDetectStorage(config.storage);
     // Process features configuration
@@ -287,14 +280,13 @@ function logConfigurationSummary(config) {
     console.log('================================\n');
 }
 /**
- * Create embedding function with specified precision
- * This ensures the model precision is respected
+ * Create embedding function (always Q8 WASM)
  */
-export async function createEmbeddingFunctionWithPrecision(precision) {
+export async function createEmbeddingFunctionWithPrecision() {
     const { createEmbeddingFunction } = await import('../utils/embedding.js');
-    // Create embedding function with specified precision
+    // Create embedding function - always Q8 WASM
     return createEmbeddingFunction({
-        precision: precision,
+        precision: 'q8',
         verbose: false // Silent by default in zero-config
     });
 }

package/dist/types/brainy.types.d.ts

@@ -518,11 +518,6 @@ export interface BrainyConfig {
         options?: any;
         branch?: string;
     };
-    model?: {
-        type: 'fast' | 'accurate' | 'balanced' | 'custom';
-        name?: string;
-        precision?: 'q8';
-    };
     index?: {
         m?: number;
         efConstruction?: number;

package/dist/utils/metadataIndex.d.ts

@@ -424,6 +424,28 @@ export declare class MetadataIndexManager {
      * Gracefully handles missing registry (first run or corrupted data).
      */
     private loadFieldRegistry;
+    /**
+     * Get list of persisted fields from storage (not in-memory)
+     * v6.7.0: Used during rebuild to discover which chunk files need deletion
+     *
+     * @returns Array of field names that have persisted sparse indices
+     */
+    private getPersistedFieldList;
+    /**
+     * Delete all chunk files for a specific field
+     * v6.7.0: Used during rebuild to ensure clean slate
+     *
+     * @param field Field name whose chunks should be deleted
+     */
+    private deleteFieldChunks;
+    /**
+     * Clear ALL metadata index data from storage (for recovery)
+     * v6.7.0: Nuclear option for recovering from corrupted index state
+     *
+     * WARNING: This deletes all indexed data - requires full rebuild after!
+     * Use when index is corrupted beyond normal rebuild repair.
+     */
+    clearAllIndexData(): Promise<void>;
     /**
      * Get count of entities by type - O(1) operation using existing tracking
      * This exposes the production-ready counting that's already maintained

package/dist/utils/metadataIndex.js

@@ -957,6 +957,11 @@ export class MetadataIndexManager {
      */
     async addToIndex(id, entityOrMetadata, skipFlush = false) {
         const fields = this.extractIndexableFields(entityOrMetadata);
+        // v6.7.0: Sanity check for excessive indexed fields (indicates possible data issue)
+        if (fields.length > 100) {
+            prodLog.warn(`Entity ${id} has ${fields.length} indexed fields (expected ~30). ` +
+                `Possible deeply nested metadata or data issue. First 10 fields: ${fields.slice(0, 10).map(f => f.field).join(', ')}`);
+        }
         // Sort fields to process 'noun' field first for type-field affinity tracking
         fields.sort((a, b) => {
             if (a.field === 'noun')
@@ -1875,6 +1880,91 @@ export class MetadataIndexManager {
             prodLog.debug('Could not load field registry:', error);
         }
     }
+    /**
+     * Get list of persisted fields from storage (not in-memory)
+     * v6.7.0: Used during rebuild to discover which chunk files need deletion
+     *
+     * @returns Array of field names that have persisted sparse indices
+     */
+    async getPersistedFieldList() {
+        try {
+            const registry = await this.storage.getMetadata('__metadata_field_registry__');
+            if (!registry?.fields || !Array.isArray(registry.fields)) {
+                return [];
+            }
+            return registry.fields.filter((f) => typeof f === 'string' && f.length > 0);
+        }
+        catch (error) {
+            prodLog.debug('Could not load persisted field list:', error);
+            return [];
+        }
+    }
+    /**
+     * Delete all chunk files for a specific field
+     * v6.7.0: Used during rebuild to ensure clean slate
+     *
+     * @param field Field name whose chunks should be deleted
+     */
+    async deleteFieldChunks(field) {
+        try {
+            // Load sparse index to get chunk IDs
+            const indexPath = `__sparse_index__${field}`;
+            const sparseData = await this.storage.getMetadata(indexPath);
+            if (sparseData) {
+                const sparseIndex = SparseIndex.fromJSON(sparseData);
+                // Delete all chunk files for this field
+                for (const chunkId of sparseIndex.getAllChunkIds()) {
+                    await this.chunkManager.deleteChunk(field, chunkId);
+                }
+                // Delete the sparse index file itself
+                await this.storage.saveMetadata(indexPath, null);
+            }
+        }
+        catch (error) {
+            // Silent failure - if we can't delete old chunks, rebuild will still work
+            // (new chunks will be created, old ones become orphaned)
+            prodLog.debug(`Could not clear chunks for field '${field}':`, error);
+        }
+    }
+    /**
+     * Clear ALL metadata index data from storage (for recovery)
+     * v6.7.0: Nuclear option for recovering from corrupted index state
+     *
+     * WARNING: This deletes all indexed data - requires full rebuild after!
+     * Use when index is corrupted beyond normal rebuild repair.
+     */
+    async clearAllIndexData() {
+        prodLog.warn('🗑️ Clearing ALL metadata index data from storage...');
+        // Get all persisted fields
+        const fields = await this.getPersistedFieldList();
+        // Delete chunks and sparse indices for each field
+        let deletedCount = 0;
+        for (const field of fields) {
+            await this.deleteFieldChunks(field);
+            deletedCount++;
+        }
+        // Delete field registry
+        try {
+            await this.storage.saveMetadata('__metadata_field_registry__', null);
+        }
+        catch (error) {
+            prodLog.debug('Could not delete field registry:', error);
+        }
+        // Clear in-memory state
+        this.fieldIndexes.clear();
+        this.dirtyFields.clear();
+        this.unifiedCache.clear('metadata');
+        this.totalEntitiesByType.clear();
+        this.entityCountsByTypeFixed.fill(0);
+        this.verbCountsByTypeFixed.fill(0);
+        this.typeFieldAffinity.clear();
+        // Clear EntityIdMapper
+        await this.idMapper.clear();
+        // Clear chunk manager cache
+        this.chunkManager.clearCache();
+        prodLog.info(`✅ Cleared ${deletedCount} field indexes and all in-memory state`);
+        prodLog.info('⚠️ Run brain.index.rebuild() to recreate the index from entity data');
+    }
     /**
      * Get count of entities by type - O(1) operation using existing tracking
      * This exposes the production-ready counting that's already maintained
@@ -2080,6 +2170,15 @@ export class MetadataIndexManager {
                 }
             }
         }
+        // v6.7.0: Sanity check for index corruption (77x overcounting bug detection)
+        const entityCount = this.idMapper.size;
+        if (entityCount > 0) {
+            const avgIdsPerEntity = totalIds / entityCount;
+            if (avgIdsPerEntity > 100) {
+                prodLog.warn(`⚠️ Metadata index may be corrupted: ${avgIdsPerEntity.toFixed(1)} avg entries/entity (expected ~30). ` +
+                    `Try running brain.index.clearAllIndexData() followed by brain.index.rebuild() to fix.`);
+            }
+        }
         return {
             totalEntries,
             totalIds,
@@ -2114,6 +2213,28 @@ export class MetadataIndexManager {
             // Clear all cached sparse indices in UnifiedCache
             // This ensures rebuild starts fresh (v3.44.1)
             this.unifiedCache.clear('metadata');
+            // v6.7.0: CRITICAL FIX - Delete existing chunk files from storage
+            // Without this, old chunk data accumulates with each rebuild causing 77x overcounting!
+            // Previous fix (v6.2.4) cleared type counts but missed chunk file accumulation.
+            prodLog.info('🗑️ Clearing existing metadata index chunks from storage...');
+            const existingFields = await this.getPersistedFieldList();
+            if (existingFields.length > 0) {
+                for (const field of existingFields) {
+                    await this.deleteFieldChunks(field);
+                }
+                // Delete field registry (will be recreated on flush)
+                try {
+                    await this.storage.saveMetadata('__metadata_field_registry__', null);
+                }
+                catch (error) {
+                    prodLog.debug('Could not delete field registry:', error);
+                }
+                prodLog.info(`✅ Cleared ${existingFields.length} field indexes from storage`);
+            }
+            // Clear EntityIdMapper to start fresh (v6.7.0)
+            await this.idMapper.clear();
+            // Clear chunk manager cache
+            this.chunkManager.clearCache();
             // Adaptive rebuild strategy based on storage adapter (v4.2.3)
             // FileSystem/Memory/OPFS: Load all at once (avoids getAllShardedFiles() overhead on every batch)
             // Cloud (GCS/S3/R2): Use pagination with small batches (prevent socket exhaustion)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "6.6.0",
+  "version": "6.6.2",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",