@soulcraft/brainy 2.15.0 → 3.0.1

package/dist/utils/typeValidation.js

@@ -124,4 +124,251 @@ export function resetValidationStats() {
         suggestions: 0
     };
 }
+// ================================================================
+// INPUT VALIDATION UTILITIES
+// ================================================================
+// Comprehensive validation for all public API parameters
+// Extends the existing type validation system
+export class ValidationError extends Error {
+    constructor(parameter, value, constraint) {
+        super(`Invalid ${parameter}: ${constraint}`);
+        this.parameter = parameter;
+        this.value = value;
+        this.constraint = constraint;
+        this.name = 'ValidationError';
+    }
+}
+/**
+ * Validate required ID parameter
+ * Standard validation for all ID-based operations
+ */
+export function validateId(id, paramName = 'id') {
+    if (id === null || id === undefined) {
+        throw new ValidationError(paramName, id, 'cannot be null or undefined');
+    }
+    if (typeof id !== 'string') {
+        throw new ValidationError(paramName, id, 'must be a string');
+    }
+    if (id.trim().length === 0) {
+        throw new ValidationError(paramName, id, 'cannot be empty string');
+    }
+    if (id.length > 512) {
+        throw new ValidationError(paramName, id, 'cannot exceed 512 characters');
+    }
+    return id.trim();
+}
+/**
+ * Validate search query input
+ * Handles string queries, vectors, and objects for search operations
+ */
+export function validateSearchQuery(query, paramName = 'query') {
+    if (query === null || query === undefined) {
+        throw new ValidationError(paramName, query, 'cannot be null or undefined');
+    }
+    // Allow strings, arrays (vectors), or objects
+    if (typeof query === 'string') {
+        if (query.trim().length === 0) {
+            throw new ValidationError(paramName, query, 'query string cannot be empty');
+        }
+        if (query.length > 10000) {
+            throw new ValidationError(paramName, query, 'query string too long (max 10000 characters)');
+        }
+        return query.trim();
+    }
+    if (Array.isArray(query)) {
+        if (query.length === 0) {
+            throw new ValidationError(paramName, query, 'array cannot be empty');
+        }
+        // Validate vector arrays contain only numbers
+        if (query.every(item => typeof item === 'number')) {
+            if (query.some(num => !isFinite(num))) {
+                throw new ValidationError(paramName, query, 'vector contains invalid numbers (NaN or Infinity)');
+            }
+        }
+        return query;
+    }
+    if (typeof query === 'object') {
+        return query;
+    }
+    throw new ValidationError(paramName, query, 'must be string, array, or object');
+}
+/**
+ * Validate data input for addNoun/updateNoun operations
+ * Handles vectors, objects, strings, and validates structure
+ */
+export function validateDataInput(data, paramName = 'data', allowNull = false) {
+    // Handle null/undefined
+    if (data === null) {
+        if (!allowNull) {
+            throw new ValidationError(paramName, data, 'Input cannot be null or undefined');
+        }
+        return data;
+    }
+    if (data === undefined) {
+        throw new ValidationError(paramName, data, 'Input cannot be null or undefined');
+    }
+    // Handle strings (including empty strings which are valid)
+    if (typeof data === 'string') {
+        // Empty strings are valid - they get converted to embeddings
+        // This matches the behavior in the embed function
+        if (data.length > 1000000) {
+            throw new ValidationError(paramName, data, 'string too long (max 1MB)');
+        }
+        return data;
+    }
+    // Handle arrays (vectors)
+    if (Array.isArray(data)) {
+        if (data.length === 0) {
+            throw new ValidationError(paramName, data, 'array cannot be empty');
+        }
+        if (data.length > 100000) {
+            throw new ValidationError(paramName, data, 'array too large (max 100k elements)');
+        }
+        // Validate vector arrays contain only numbers
+        if (data.every(item => typeof item === 'number')) {
+            if (data.some(num => !isFinite(num))) {
+                throw new ValidationError(paramName, data, 'vector contains invalid numbers (NaN or Infinity)');
+            }
+        }
+        return data;
+    }
+    // Handle objects
+    if (typeof data === 'object') {
+        try {
+            // Quick check if object can be serialized (avoids circular references)
+            JSON.stringify(data);
+        }
+        catch (error) {
+            throw new ValidationError(paramName, data, 'object contains circular references or unserializable values');
+        }
+        return data;
+    }
+    // Handle primitive types
+    if (typeof data === 'number') {
+        if (!isFinite(data)) {
+            throw new ValidationError(paramName, data, 'number must be finite (not NaN or Infinity)');
+        }
+        return data;
+    }
+    if (typeof data === 'boolean') {
+        return data;
+    }
+    throw new ValidationError(paramName, data, 'must be string, number, boolean, array, or object');
+}
+/**
+ * Validate search options
+ * Comprehensive validation for search API options
+ */
+export function validateSearchOptions(options, paramName = 'options') {
+    if (options === null || options === undefined) {
+        return {}; // Default to empty options
+    }
+    if (typeof options !== 'object' || Array.isArray(options)) {
+        throw new ValidationError(paramName, options, 'must be an object');
+    }
+    const opts = options;
+    // Validate limit
+    if ('limit' in opts) {
+        const limit = opts.limit;
+        if (typeof limit !== 'number' || limit < 1 || limit > 10000 || !Number.isInteger(limit)) {
+            throw new ValidationError(`${paramName}.limit`, limit, 'must be integer between 1 and 10000');
+        }
+    }
+    // Validate offset  
+    if ('offset' in opts) {
+        const offset = opts.offset;
+        if (typeof offset !== 'number' || offset < 0 || !Number.isInteger(offset)) {
+            throw new ValidationError(`${paramName}.offset`, offset, 'must be non-negative integer');
+        }
+    }
+    // Validate threshold
+    if ('threshold' in opts) {
+        const threshold = opts.threshold;
+        if (typeof threshold !== 'number' || threshold < 0 || threshold > 1) {
+            throw new ValidationError(`${paramName}.threshold`, threshold, 'must be number between 0 and 1');
+        }
+    }
+    // Validate timeout
+    if ('timeout' in opts) {
+        const timeout = opts.timeout;
+        if (typeof timeout !== 'number' || timeout < 1 || timeout > 300000 || !Number.isInteger(timeout)) {
+            throw new ValidationError(`${paramName}.timeout`, timeout, 'must be integer between 1 and 300000 milliseconds');
+        }
+    }
+    // Validate nounTypes array
+    if ('nounTypes' in opts) {
+        if (!Array.isArray(opts.nounTypes)) {
+            throw new ValidationError(`${paramName}.nounTypes`, opts.nounTypes, 'must be an array');
+        }
+        if (opts.nounTypes.length > 100) {
+            throw new ValidationError(`${paramName}.nounTypes`, opts.nounTypes, 'too many noun types (max 100)');
+        }
+        // Validate each noun type
+        opts.nounTypes = opts.nounTypes.map((type, index) => {
+            try {
+                return validateNounType(type);
+            }
+            catch (error) {
+                if (error instanceof Error) {
+                    throw new ValidationError(`${paramName}.nounTypes[${index}]`, type, error.message);
+                }
+                throw error;
+            }
+        });
+    }
+    // Validate itemIds array
+    if ('itemIds' in opts) {
+        if (!Array.isArray(opts.itemIds)) {
+            throw new ValidationError(`${paramName}.itemIds`, opts.itemIds, 'must be an array');
+        }
+        if (opts.itemIds.length > 10000) {
+            throw new ValidationError(`${paramName}.itemIds`, opts.itemIds, 'too many item IDs (max 10000)');
+        }
+        opts.itemIds = opts.itemIds.map((id, index) => {
+            try {
+                return validateId(id, `${paramName}.itemIds[${index}]`);
+            }
+            catch (error) {
+                throw error; // Re-throw with proper context
+            }
+        });
+    }
+    return opts;
+}
+/**
+ * Validate ID arrays (for bulk operations)
+ */
+export function validateIdArray(ids, paramName = 'ids') {
+    if (ids === null || ids === undefined) {
+        throw new ValidationError(paramName, ids, 'cannot be null or undefined');
+    }
+    if (!Array.isArray(ids)) {
+        throw new ValidationError(paramName, ids, 'must be an array');
+    }
+    if (ids.length === 0) {
+        throw new ValidationError(paramName, ids, 'cannot be empty');
+    }
+    if (ids.length > 10000) {
+        throw new ValidationError(paramName, ids, 'too large (max 10000 items)');
+    }
+    return ids.map((id, index) => {
+        try {
+            return validateId(id, `${paramName}[${index}]`);
+        }
+        catch (error) {
+            throw error; // Re-throw with proper array context
+        }
+    });
+}
+/**
+ * Track validation stats for monitoring
+ */
+export function recordValidation(success) {
+    if (success) {
+        stats.validated++;
+    }
+    else {
+        stats.failed++;
+    }
+}
 //# sourceMappingURL=typeValidation.js.map

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "2.15.0",
+  "version": "3.0.1",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",
   "bin": {
-    "brainy": "./bin/brainy.js"
+    "brainy": "bin/brainy.js"
   },
   "sideEffects": [
     "./dist/setup.js",
@@ -68,19 +68,22 @@
     "test:coverage": "vitest run --config tests/configs/vitest.unit.config.ts --coverage",
     "test:unit": "vitest run --config tests/configs/vitest.unit.config.ts",
     "test:integration": "NODE_OPTIONS='--max-old-space-size=32768' vitest run --config tests/configs/vitest.integration.config.ts",
+    "test:s3": "vitest run tests/integration/s3-storage.test.ts",
+    "test:distributed": "vitest run tests/integration/distributed.test.ts",
+    "test:cloud": "npm run test:s3 && npm run test:distributed",
     "test:all": "npm run test:unit && npm run test:integration",
     "test:ci-unit": "CI=true vitest run --config tests/configs/vitest.unit.config.ts",
     "test:ci-integration": "NODE_OPTIONS='--max-old-space-size=16384' CI=true vitest run --config tests/configs/vitest.integration.config.ts",
     "test:ci": "npm run test:ci-unit",
     "download-models": "node scripts/download-models.cjs",
-    "download-models:fp32": "node scripts/download-models.cjs fp32",
-    "download-models:q8": "node scripts/download-models.cjs q8",
-    "download-models:both": "node scripts/download-models.cjs",
+    "download-models:q8": "node scripts/download-models.cjs",
     "models:verify": "node scripts/ensure-models.js",
     "lint": "eslint --ext .ts,.js src/",
     "lint:fix": "eslint --ext .ts,.js src/ --fix",
     "format": "prettier --write \"src/**/*.{ts,js}\"",
     "format:check": "prettier --check \"src/**/*.{ts,js}\"",
+    "migrate:logger": "tsx scripts/migrate-to-structured-logger.ts",
+    "migrate:logger:dry": "tsx scripts/migrate-to-structured-logger.ts --dry-run",
     "release": "standard-version",
     "release:patch": "standard-version --release-as patch",
     "release:minor": "standard-version --release-as minor",
@@ -133,12 +136,16 @@
     "@rollup/plugin-node-resolve": "^16.0.1",
     "@rollup/plugin-replace": "^6.0.2",
     "@rollup/plugin-terser": "^0.4.4",
+    "@testcontainers/redis": "^11.5.1",
     "@types/node": "^20.11.30",
     "@types/uuid": "^10.0.0",
+    "@types/ws": "^8.18.1",
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "@typescript-eslint/parser": "^8.0.0",
     "@vitest/coverage-v8": "^3.2.4",
+    "minio": "^8.0.5",
     "standard-version": "^9.5.0",
+    "testcontainers": "^11.5.1",
     "tsx": "^4.19.2",
     "typescript": "^5.4.5",
     "vitest": "^3.2.4"
@@ -153,7 +160,8 @@
     "inquirer": "^12.9.3",
     "ora": "^8.2.0",
     "prompts": "^2.4.2",
-    "uuid": "^9.0.1"
+    "uuid": "^9.0.1",
+    "ws": "^8.18.3"
   },
   "prettier": {
     "arrowParens": "always",

package/scripts/download-models.cjs

@@ -9,10 +9,8 @@ const path = require('path')
 const MODEL_NAME = 'Xenova/all-MiniLM-L6-v2'
 const OUTPUT_DIR = './models'
 
-// Parse command line arguments for model type selection
-const args = process.argv.slice(2)
-const downloadType = args.includes('fp32') ? 'fp32' : 
-                    args.includes('q8') ? 'q8' : 'both'
+// Always download Q8 model only
+const downloadType = 'q8'
 
 async function downloadModels() {
   // Use dynamic import for ES modules in CommonJS
@@ -26,23 +24,16 @@ async function downloadModels() {
     console.log('🧠 Brainy Model Downloader v2.8.0')
     console.log('===================================')
     console.log(`   Model: ${MODEL_NAME}`)
-    console.log(`   Type: ${downloadType} (fp32, q8, or both)`)
+    console.log(`   Type: Q8 (optimized, 99% accuracy)`)
     console.log(`   Cache: ${env.cacheDir}`)
     console.log('')
     
     // Create output directory
     await fs.mkdir(OUTPUT_DIR, { recursive: true })
     
-    // Download models based on type
-    if (downloadType === 'both' || downloadType === 'fp32') {
-      console.log('📥 Downloading FP32 model (full precision, 90MB)...')
-      await downloadModelVariant('fp32')
-    }
-    
-    if (downloadType === 'both' || downloadType === 'q8') {
-      console.log('📥 Downloading Q8 model (quantized, 23MB)...')
-      await downloadModelVariant('q8')
-    }
+    // Download Q8 model only
+    console.log('📥 Downloading Q8 model (quantized, 33MB, 99% accuracy)...')
+    await downloadModelVariant('q8')
     
     // Copy ALL model files from cache to our models directory
     console.log('📋 Copying model files to bundle directory...')

package/dist/augmentations/walAugmentation.d.ts

@@ -1,111 +0,0 @@
-/**
- * Write-Ahead Log (WAL) Augmentation
- *
- * Provides file-based durability and atomicity for storage operations
- * Automatically enabled for all critical storage operations
- *
- * Features:
- * - True file-based persistence for crash recovery
- * - Operation replay after startup
- * - Automatic log rotation and cleanup
- * - Cross-platform compatibility (filesystem, OPFS, cloud)
- */
-import { BaseAugmentation } from './brainyAugmentation.js';
-interface WALConfig {
-    enabled?: boolean;
-    immediateWrites?: boolean;
-    adaptivePersistence?: boolean;
-    walPrefix?: string;
-    maxSize?: number;
-    checkpointInterval?: number;
-    autoRecover?: boolean;
-    maxRetries?: number;
-}
-export declare class WALAugmentation extends BaseAugmentation {
-    name: string;
-    timing: "around";
-    metadata: "readonly";
-    operations: ("addNoun" | "addVerb" | "saveNoun" | "saveVerb" | "updateMetadata" | "delete" | "deleteVerb" | "clear")[];
-    priority: number;
-    readonly category: "internal";
-    readonly description = "Write-ahead logging for durability and crash recovery";
-    private config;
-    private currentLogId;
-    private operationCounter;
-    private checkpointTimer?;
-    private isRecovering;
-    constructor(config?: WALConfig);
-    protected onInitialize(): Promise<void>;
-    shouldExecute(operation: string, params: any): boolean;
-    execute<T = any>(operation: string, params: any, next: () => Promise<T>): Promise<T>;
-    /**
-     * Asynchronous WAL entry logging (fire-and-forget for immediate writes)
-     */
-    private logAsyncWALEntry;
-    /**
-     * Write WAL entry to persistent storage using storage adapter
-     */
-    private writeWALEntry;
-    /**
-     * Recover pending operations from all existing WAL files
-     */
-    private recoverPendingOperations;
-    /**
-     * Find all WAL files in storage
-     */
-    private findWALFiles;
-    /**
-     * Read WAL entries from a file
-     */
-    private readWALEntries;
-    /**
-     * Find operations that were started but not completed
-     */
-    private findPendingOperations;
-    /**
-     * Replay an operation during recovery
-     */
-    private replayOperation;
-    /**
-     * Create a checkpoint to mark a point in time
-     */
-    private createCheckpoint;
-    /**
-     * Check if log rotation is needed
-     */
-    private checkLogRotation;
-    /**
-     * Sanitize parameters for logging (remove large objects)
-     */
-    private sanitizeParams;
-    /**
-     * Get WAL statistics
-     */
-    getStats(): {
-        enabled: boolean;
-        currentLogId: string;
-        operationCount: number;
-        logSize: number;
-        pendingOperations: number;
-        failedOperations: number;
-    };
-    /**
-     * Manually trigger checkpoint
-     */
-    checkpoint(): Promise<void>;
-    /**
-     * Manually trigger log rotation
-     */
-    rotate(): Promise<void>;
-    /**
-     * Write WAL data directly to storage without embedding
-     * This bypasses the brain's AI processing pipeline for raw WAL data
-     */
-    private writeWALFileDirectly;
-    /**
-     * Read WAL data directly from storage without embedding
-     */
-    private readWALFileDirectly;
-    protected onShutdown(): Promise<void>;
-}
-export {};