@soulcraft/brainy 3.0.0 → 3.1.0

package/dist/utils/typeValidation.js

@@ -0,0 +1,374 @@
+import { NounType, VerbType } from '../types/graphTypes.js';
+// Type sets for O(1) validation
+const VALID_NOUN_TYPES = new Set(Object.values(NounType));
+const VALID_VERB_TYPES = new Set(Object.values(VerbType));
+// Type guards
+export function isValidNounType(type) {
+    return typeof type === 'string' && VALID_NOUN_TYPES.has(type);
+}
+export function isValidVerbType(type) {
+    return typeof type === 'string' && VALID_VERB_TYPES.has(type);
+}
+// Validators with helpful errors
+export function validateNounType(type) {
+    if (!isValidNounType(type)) {
+        const suggestion = findClosestMatch(String(type), VALID_NOUN_TYPES);
+        throw new Error(`Invalid noun type: '${type}'. ${suggestion ? `Did you mean '${suggestion}'?` : ''} ` +
+            `Valid types are: ${[...VALID_NOUN_TYPES].sort().join(', ')}`);
+    }
+    return type;
+}
+export function validateVerbType(type) {
+    if (!isValidVerbType(type)) {
+        const suggestion = findClosestMatch(String(type), VALID_VERB_TYPES);
+        throw new Error(`Invalid verb type: '${type}'. ${suggestion ? `Did you mean '${suggestion}'?` : ''} ` +
+            `Valid types are: ${[...VALID_VERB_TYPES].sort().join(', ')}`);
+    }
+    return type;
+}
+export function validateGraphNoun(noun) {
+    if (!noun || typeof noun !== 'object') {
+        throw new Error('Invalid noun: must be an object');
+    }
+    const n = noun;
+    if (!n.noun) {
+        throw new Error('Invalid noun: missing required "noun" type field');
+    }
+    n.noun = validateNounType(n.noun);
+    return n;
+}
+export function validateGraphVerb(verb) {
+    if (!verb || typeof verb !== 'object') {
+        throw new Error('Invalid verb: must be an object');
+    }
+    const v = verb;
+    if (!v.verb) {
+        throw new Error('Invalid verb: missing required "verb" type field');
+    }
+    v.verb = validateVerbType(v.verb);
+    return v;
+}
+// Helper for suggestions using Levenshtein distance
+function findClosestMatch(input, validSet) {
+    if (!input)
+        return null;
+    const lower = input.toLowerCase();
+    let bestMatch = null;
+    let bestScore = Infinity;
+    for (const valid of validSet) {
+        const validLower = valid.toLowerCase();
+        // Exact match (case-insensitive)
+        if (validLower === lower) {
+            return valid;
+        }
+        // Substring match
+        if (validLower.includes(lower) || lower.includes(validLower)) {
+            return valid;
+        }
+        // Calculate Levenshtein distance
+        const distance = levenshteinDistance(lower, validLower);
+        if (distance < bestScore && distance <= 3) { // Threshold of 3 for suggestions
+            bestScore = distance;
+            bestMatch = valid;
+        }
+    }
+    return bestMatch;
+}
+// Levenshtein distance implementation
+function levenshteinDistance(str1, str2) {
+    const m = str1.length;
+    const n = str2.length;
+    const dp = Array(m + 1).fill(null).map(() => Array(n + 1).fill(0));
+    for (let i = 0; i <= m; i++) {
+        dp[i][0] = i;
+    }
+    for (let j = 0; j <= n; j++) {
+        dp[0][j] = j;
+    }
+    for (let i = 1; i <= m; i++) {
+        for (let j = 1; j <= n; j++) {
+            if (str1[i - 1] === str2[j - 1]) {
+                dp[i][j] = dp[i - 1][j - 1];
+            }
+            else {
+                dp[i][j] = 1 + Math.min(dp[i - 1][j], // deletion
+                dp[i][j - 1], // insertion
+                dp[i - 1][j - 1] // substitution
+                );
+            }
+        }
+    }
+    return dp[m][n];
+}
+// Batch validation helpers
+export function validateNounTypes(types) {
+    return types.map(validateNounType);
+}
+export function validateVerbTypes(types) {
+    return types.map(validateVerbType);
+}
+let stats = {
+    validated: 0,
+    failed: 0,
+    inferred: 0,
+    suggestions: 0
+};
+export function getValidationStats() {
+    return { ...stats };
+}
+export function resetValidationStats() {
+    stats = {
+        validated: 0,
+        failed: 0,
+        inferred: 0,
+        suggestions: 0
+    };
+}
+// ================================================================
+// INPUT VALIDATION UTILITIES
+// ================================================================
+// Comprehensive validation for all public API parameters
+// Extends the existing type validation system
+export class ValidationError extends Error {
+    constructor(parameter, value, constraint) {
+        super(`Invalid ${parameter}: ${constraint}`);
+        this.parameter = parameter;
+        this.value = value;
+        this.constraint = constraint;
+        this.name = 'ValidationError';
+    }
+}
+/**
+ * Validate required ID parameter
+ * Standard validation for all ID-based operations
+ */
+export function validateId(id, paramName = 'id') {
+    if (id === null || id === undefined) {
+        throw new ValidationError(paramName, id, 'cannot be null or undefined');
+    }
+    if (typeof id !== 'string') {
+        throw new ValidationError(paramName, id, 'must be a string');
+    }
+    if (id.trim().length === 0) {
+        throw new ValidationError(paramName, id, 'cannot be empty string');
+    }
+    if (id.length > 512) {
+        throw new ValidationError(paramName, id, 'cannot exceed 512 characters');
+    }
+    return id.trim();
+}
+/**
+ * Validate search query input
+ * Handles string queries, vectors, and objects for search operations
+ */
+export function validateSearchQuery(query, paramName = 'query') {
+    if (query === null || query === undefined) {
+        throw new ValidationError(paramName, query, 'cannot be null or undefined');
+    }
+    // Allow strings, arrays (vectors), or objects
+    if (typeof query === 'string') {
+        if (query.trim().length === 0) {
+            throw new ValidationError(paramName, query, 'query string cannot be empty');
+        }
+        if (query.length > 10000) {
+            throw new ValidationError(paramName, query, 'query string too long (max 10000 characters)');
+        }
+        return query.trim();
+    }
+    if (Array.isArray(query)) {
+        if (query.length === 0) {
+            throw new ValidationError(paramName, query, 'array cannot be empty');
+        }
+        // Validate vector arrays contain only numbers
+        if (query.every(item => typeof item === 'number')) {
+            if (query.some(num => !isFinite(num))) {
+                throw new ValidationError(paramName, query, 'vector contains invalid numbers (NaN or Infinity)');
+            }
+        }
+        return query;
+    }
+    if (typeof query === 'object') {
+        return query;
+    }
+    throw new ValidationError(paramName, query, 'must be string, array, or object');
+}
+/**
+ * Validate data input for addNoun/updateNoun operations
+ * Handles vectors, objects, strings, and validates structure
+ */
+export function validateDataInput(data, paramName = 'data', allowNull = false) {
+    // Handle null/undefined
+    if (data === null) {
+        if (!allowNull) {
+            throw new ValidationError(paramName, data, 'Input cannot be null or undefined');
+        }
+        return data;
+    }
+    if (data === undefined) {
+        throw new ValidationError(paramName, data, 'Input cannot be null or undefined');
+    }
+    // Handle strings (including empty strings which are valid)
+    if (typeof data === 'string') {
+        // Empty strings are valid - they get converted to embeddings
+        // This matches the behavior in the embed function
+        if (data.length > 1000000) {
+            throw new ValidationError(paramName, data, 'string too long (max 1MB)');
+        }
+        return data;
+    }
+    // Handle arrays (vectors)
+    if (Array.isArray(data)) {
+        if (data.length === 0) {
+            throw new ValidationError(paramName, data, 'array cannot be empty');
+        }
+        if (data.length > 100000) {
+            throw new ValidationError(paramName, data, 'array too large (max 100k elements)');
+        }
+        // Validate vector arrays contain only numbers
+        if (data.every(item => typeof item === 'number')) {
+            if (data.some(num => !isFinite(num))) {
+                throw new ValidationError(paramName, data, 'vector contains invalid numbers (NaN or Infinity)');
+            }
+        }
+        return data;
+    }
+    // Handle objects
+    if (typeof data === 'object') {
+        try {
+            // Quick check if object can be serialized (avoids circular references)
+            JSON.stringify(data);
+        }
+        catch (error) {
+            throw new ValidationError(paramName, data, 'object contains circular references or unserializable values');
+        }
+        return data;
+    }
+    // Handle primitive types
+    if (typeof data === 'number') {
+        if (!isFinite(data)) {
+            throw new ValidationError(paramName, data, 'number must be finite (not NaN or Infinity)');
+        }
+        return data;
+    }
+    if (typeof data === 'boolean') {
+        return data;
+    }
+    throw new ValidationError(paramName, data, 'must be string, number, boolean, array, or object');
+}
+/**
+ * Validate search options
+ * Comprehensive validation for search API options
+ */
+export function validateSearchOptions(options, paramName = 'options') {
+    if (options === null || options === undefined) {
+        return {}; // Default to empty options
+    }
+    if (typeof options !== 'object' || Array.isArray(options)) {
+        throw new ValidationError(paramName, options, 'must be an object');
+    }
+    const opts = options;
+    // Validate limit
+    if ('limit' in opts) {
+        const limit = opts.limit;
+        if (typeof limit !== 'number' || limit < 1 || limit > 10000 || !Number.isInteger(limit)) {
+            throw new ValidationError(`${paramName}.limit`, limit, 'must be integer between 1 and 10000');
+        }
+    }
+    // Validate offset  
+    if ('offset' in opts) {
+        const offset = opts.offset;
+        if (typeof offset !== 'number' || offset < 0 || !Number.isInteger(offset)) {
+            throw new ValidationError(`${paramName}.offset`, offset, 'must be non-negative integer');
+        }
+    }
+    // Validate threshold
+    if ('threshold' in opts) {
+        const threshold = opts.threshold;
+        if (typeof threshold !== 'number' || threshold < 0 || threshold > 1) {
+            throw new ValidationError(`${paramName}.threshold`, threshold, 'must be number between 0 and 1');
+        }
+    }
+    // Validate timeout
+    if ('timeout' in opts) {
+        const timeout = opts.timeout;
+        if (typeof timeout !== 'number' || timeout < 1 || timeout > 300000 || !Number.isInteger(timeout)) {
+            throw new ValidationError(`${paramName}.timeout`, timeout, 'must be integer between 1 and 300000 milliseconds');
+        }
+    }
+    // Validate nounTypes array
+    if ('nounTypes' in opts) {
+        if (!Array.isArray(opts.nounTypes)) {
+            throw new ValidationError(`${paramName}.nounTypes`, opts.nounTypes, 'must be an array');
+        }
+        if (opts.nounTypes.length > 100) {
+            throw new ValidationError(`${paramName}.nounTypes`, opts.nounTypes, 'too many noun types (max 100)');
+        }
+        // Validate each noun type
+        opts.nounTypes = opts.nounTypes.map((type, index) => {
+            try {
+                return validateNounType(type);
+            }
+            catch (error) {
+                if (error instanceof Error) {
+                    throw new ValidationError(`${paramName}.nounTypes[${index}]`, type, error.message);
+                }
+                throw error;
+            }
+        });
+    }
+    // Validate itemIds array
+    if ('itemIds' in opts) {
+        if (!Array.isArray(opts.itemIds)) {
+            throw new ValidationError(`${paramName}.itemIds`, opts.itemIds, 'must be an array');
+        }
+        if (opts.itemIds.length > 10000) {
+            throw new ValidationError(`${paramName}.itemIds`, opts.itemIds, 'too many item IDs (max 10000)');
+        }
+        opts.itemIds = opts.itemIds.map((id, index) => {
+            try {
+                return validateId(id, `${paramName}.itemIds[${index}]`);
+            }
+            catch (error) {
+                throw error; // Re-throw with proper context
+            }
+        });
+    }
+    return opts;
+}
+/**
+ * Validate ID arrays (for bulk operations)
+ */
+export function validateIdArray(ids, paramName = 'ids') {
+    if (ids === null || ids === undefined) {
+        throw new ValidationError(paramName, ids, 'cannot be null or undefined');
+    }
+    if (!Array.isArray(ids)) {
+        throw new ValidationError(paramName, ids, 'must be an array');
+    }
+    if (ids.length === 0) {
+        throw new ValidationError(paramName, ids, 'cannot be empty');
+    }
+    if (ids.length > 10000) {
+        throw new ValidationError(paramName, ids, 'too large (max 10000 items)');
+    }
+    return ids.map((id, index) => {
+        try {
+            return validateId(id, `${paramName}[${index}]`);
+        }
+        catch (error) {
+            throw error; // Re-throw with proper array context
+        }
+    });
+}
+/**
+ * Track validation stats for monitoring
+ */
+export function recordValidation(success) {
+    if (success) {
+        stats.validated++;
+    }
+    else {
+        stats.failed++;
+    }
+}
+//# sourceMappingURL=typeValidation.js.map

package/dist/utils/version.js

@@ -1,14 +1,30 @@
 /**
  * Version utilities for Brainy
  */
-// Package version - this should be updated during the build process
-const BRAINY_VERSION = '0.41.0';
+import { readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+// Get package.json path relative to this file
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const packageJsonPath = join(__dirname, '../../package.json');
+let cachedVersion = null;
 /**
  * Get the current Brainy package version
  * @returns The current version string
  */
 export function getBrainyVersion() {
-    return BRAINY_VERSION;
+    if (!cachedVersion) {
+        try {
+            const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8'));
+            cachedVersion = packageJson.version;
+        }
+        catch {
+            // Fallback version if package.json can't be read
+            cachedVersion = '2.7.1';
+        }
+    }
+    return cachedVersion;
 }
 /**
  * Get version information for augmentation metadata

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",
   "bin": {
-    "brainy": "./bin/brainy.js"
+    "brainy": "bin/brainy.js"
   },
   "sideEffects": [
     "./dist/setup.js",
@@ -15,17 +15,8 @@
     "./src/setup.ts",
     "./src/utils/textEncoding.ts"
   ],
-  "browser": {
-    "fs": false,
-    "fs/promises": false,
-    "path": "path-browserify",
-    "crypto": "crypto-browserify",
-    "./dist/cortex/cortex.js": "./dist/browserFramework.js"
-  },
   "exports": {
     ".": {
-      "browser": "./dist/browserFramework.js",
-      "node": "./dist/index.js",
       "import": "./dist/index.js",
       "types": "./dist/index.d.ts"
     },
@@ -45,17 +36,13 @@
       "import": "./dist/utils/textEncoding.js",
       "types": "./dist/utils/textEncoding.d.ts"
     },
-    "./browserFramework": {
-      "import": "./dist/browserFramework.js",
-      "types": "./dist/browserFramework.d.ts"
-    },
     "./universal": {
       "import": "./dist/universal/index.js",
       "types": "./dist/universal/index.d.ts"
     }
   },
   "engines": {
-    "node": ">=18.0.0"
+    "node": "22.x"
   },
   "scripts": {
     "build": "npm run build:patterns:if-needed && tsc",
@@ -68,16 +55,22 @@
     "test:coverage": "vitest run --config tests/configs/vitest.unit.config.ts --coverage",
     "test:unit": "vitest run --config tests/configs/vitest.unit.config.ts",
     "test:integration": "NODE_OPTIONS='--max-old-space-size=32768' vitest run --config tests/configs/vitest.integration.config.ts",
+    "test:s3": "vitest run tests/integration/s3-storage.test.ts",
+    "test:distributed": "vitest run tests/integration/distributed.test.ts",
+    "test:cloud": "npm run test:s3 && npm run test:distributed",
     "test:all": "npm run test:unit && npm run test:integration",
     "test:ci-unit": "CI=true vitest run --config tests/configs/vitest.unit.config.ts",
     "test:ci-integration": "NODE_OPTIONS='--max-old-space-size=16384' CI=true vitest run --config tests/configs/vitest.integration.config.ts",
     "test:ci": "npm run test:ci-unit",
     "download-models": "node scripts/download-models.cjs",
+    "download-models:q8": "node scripts/download-models.cjs",
     "models:verify": "node scripts/ensure-models.js",
     "lint": "eslint --ext .ts,.js src/",
     "lint:fix": "eslint --ext .ts,.js src/ --fix",
     "format": "prettier --write \"src/**/*.{ts,js}\"",
     "format:check": "prettier --check \"src/**/*.{ts,js}\"",
+    "migrate:logger": "tsx scripts/migrate-to-structured-logger.ts",
+    "migrate:logger:dry": "tsx scripts/migrate-to-structured-logger.ts --dry-run",
     "release": "standard-version",
     "release:patch": "standard-version --release-as patch",
     "release:minor": "standard-version --release-as minor",
@@ -130,12 +123,16 @@
     "@rollup/plugin-node-resolve": "^16.0.1",
     "@rollup/plugin-replace": "^6.0.2",
     "@rollup/plugin-terser": "^0.4.4",
+    "@testcontainers/redis": "^11.5.1",
     "@types/node": "^20.11.30",
     "@types/uuid": "^10.0.0",
+    "@types/ws": "^8.18.1",
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "@typescript-eslint/parser": "^8.0.0",
     "@vitest/coverage-v8": "^3.2.4",
+    "minio": "^8.0.5",
     "standard-version": "^9.5.0",
+    "testcontainers": "^11.5.1",
     "tsx": "^4.19.2",
     "typescript": "^5.4.5",
     "vitest": "^3.2.4"
@@ -150,7 +147,8 @@
     "inquirer": "^12.9.3",
     "ora": "^8.2.0",
     "prompts": "^2.4.2",
-    "uuid": "^9.0.1"
+    "uuid": "^9.0.1",
+    "ws": "^8.18.3"
   },
   "prettier": {
     "arrowParens": "always",

package/scripts/download-models.cjs

@@ -9,6 +9,9 @@ const path = require('path')
 const MODEL_NAME = 'Xenova/all-MiniLM-L6-v2'
 const OUTPUT_DIR = './models'
 
+// Always download Q8 model only
+const downloadType = 'q8'
+
 async function downloadModels() {
   // Use dynamic import for ES modules in CommonJS
   const { pipeline, env } = await import('@huggingface/transformers')
@@ -16,29 +19,24 @@ async function downloadModels() {
   // Configure transformers.js to use local cache
   env.cacheDir = './models-cache'
   env.allowRemoteModels = true
+  
   try {
-    console.log('🔄 Downloading all-MiniLM-L6-v2 model for offline bundling...')
+    console.log('🧠 Brainy Model Downloader v2.8.0')
+    console.log('===================================')
     console.log(`   Model: ${MODEL_NAME}`)
+    console.log(`   Type: Q8 (optimized, 99% accuracy)`)
     console.log(`   Cache: ${env.cacheDir}`)
+    console.log('')
     
     // Create output directory
     await fs.mkdir(OUTPUT_DIR, { recursive: true })
     
-    // Load the model to force download
-    console.log('📥 Loading model pipeline...')
-    const extractor = await pipeline('feature-extraction', MODEL_NAME)
-    
-    // Test the model to make sure it works
-    console.log('🧪 Testing model...')
-    const testResult = await extractor(['Hello world!'], {
-      pooling: 'mean',
-      normalize: true
-    })
-    
-    console.log(`✅ Model test successful! Embedding dimensions: ${testResult.data.length}`)
+    // Download Q8 model only
+    console.log('📥 Downloading Q8 model (quantized, 33MB, 99% accuracy)...')
+    await downloadModelVariant('q8')
     
     // Copy ALL model files from cache to our models directory
-    console.log('📋 Copying ALL model files to bundle directory...')
+    console.log('📋 Copying model files to bundle directory...')
     
     const cacheDir = path.resolve(env.cacheDir)
     const outputDir = path.resolve(OUTPUT_DIR)
@@ -62,22 +60,89 @@ async function downloadModels() {
     console.log(`   Total size: ${await calculateDirectorySize(outputDir)} MB`)
     console.log(`   Location: ${outputDir}`)
     
-    // Create a marker file
+    // Create a marker file with downloaded model info
+    const markerData = {
+      model: MODEL_NAME,
+      bundledAt: new Date().toISOString(),
+      version: '2.8.0',
+      downloadType: downloadType,
+      models: {}
+    }
+    
+    // Check which models were downloaded
+    const fp32Path = path.join(outputDir, 'Xenova/all-MiniLM-L6-v2/onnx/model.onnx')
+    const q8Path = path.join(outputDir, 'Xenova/all-MiniLM-L6-v2/onnx/model_quantized.onnx')
+    
+    if (await fileExists(fp32Path)) {
+      const stats = await fs.stat(fp32Path)
+      markerData.models.fp32 = {
+        file: 'onnx/model.onnx',
+        size: stats.size,
+        sizeFormatted: `${Math.round(stats.size / (1024 * 1024))}MB`
+      }
+    }
+    
+    if (await fileExists(q8Path)) {
+      const stats = await fs.stat(q8Path)
+      markerData.models.q8 = {
+        file: 'onnx/model_quantized.onnx', 
+        size: stats.size,
+        sizeFormatted: `${Math.round(stats.size / (1024 * 1024))}MB`
+      }
+    }
+    
     await fs.writeFile(
       path.join(outputDir, '.brainy-models-bundled'),
-      JSON.stringify({
-        model: MODEL_NAME,
-        bundledAt: new Date().toISOString(),
-        version: '1.0.0'
-      }, null, 2)
+      JSON.stringify(markerData, null, 2)
     )
     
+    console.log('')
+    console.log('✅ Download complete! Available models:')
+    if (markerData.models.fp32) {
+      console.log(`   • FP32: ${markerData.models.fp32.sizeFormatted} (full precision)`)
+    }
+    if (markerData.models.q8) {
+      console.log(`   • Q8: ${markerData.models.q8.sizeFormatted} (quantized, 75% smaller)`)
+    }
+    console.log('')
+    console.log('Air-gap deployment ready! 🚀')
+    
   } catch (error) {
     console.error('❌ Error downloading models:', error)
     process.exit(1)
   }
 }
 
+// Download a specific model variant
+async function downloadModelVariant(dtype) {
+  const { pipeline } = await import('@huggingface/transformers')
+  
+  try {
+    // Load the model to force download
+    const extractor = await pipeline('feature-extraction', MODEL_NAME, {
+      dtype: dtype,
+      cache_dir: './models-cache'
+    })
+    
+    // Test the model
+    const testResult = await extractor(['Hello world!'], {
+      pooling: 'mean', 
+      normalize: true
+    })
+    
+    console.log(`   ✅ ${dtype.toUpperCase()} model downloaded and tested (${testResult.data.length} dimensions)`)
+    
+    // Dispose to free memory
+    if (extractor.dispose) {
+      await extractor.dispose()
+    }
+    
+  } catch (error) {
+    console.error(`   ❌ Failed to download ${dtype} model:`, error)
+    throw error
+  }
+}
+
 async function findModelDirectories(baseDir, modelName) {
   const dirs = []
   
@@ -141,6 +206,15 @@ async function dirExists(dir) {
   }
 }
 
+async function fileExists(file) {
+  try {
+    const stats = await fs.stat(file)
+    return stats.isFile()
+  } catch (error) {
+    return false
+  }
+}
+
 async function copyDirectory(src, dest) {
   await fs.mkdir(dest, { recursive: true })
   const entries = await fs.readdir(src, { withFileTypes: true })