@soulcraft/brainy 1.5.0 → 2.0.0

package/dist/utils/unifiedCache.d.ts

@@ -0,0 +1,103 @@
+/**
+ * UnifiedCache - Single cache for both HNSW and MetadataIndex
+ * Prevents resource competition with cost-aware eviction
+ */
+export interface CacheItem {
+    key: string;
+    type: 'hnsw' | 'metadata' | 'embedding' | 'other';
+    data: any;
+    size: number;
+    rebuildCost: number;
+    lastAccess: number;
+    accessCount: number;
+}
+export interface UnifiedCacheConfig {
+    maxSize?: number;
+    enableRequestCoalescing?: boolean;
+    enableFairnessCheck?: boolean;
+    fairnessCheckInterval?: number;
+    persistPatterns?: boolean;
+}
+export declare class UnifiedCache {
+    private cache;
+    private access;
+    private loadingPromises;
+    private typeAccessCounts;
+    private totalAccessCount;
+    private currentSize;
+    private readonly maxSize;
+    private readonly config;
+    constructor(config?: UnifiedCacheConfig);
+    /**
+     * Get item from cache with request coalescing
+     */
+    get(key: string, loadFn?: () => Promise<any>): Promise<any>;
+    /**
+     * Set item in cache with cost-aware eviction
+     */
+    set(key: string, data: any, type: 'hnsw' | 'metadata' | 'embedding' | 'other', size: number, rebuildCost?: number): void;
+    /**
+     * Evict item with lowest value (access count / rebuild cost)
+     */
+    private evictLowestValue;
+    /**
+     * Size-aware eviction - try to match needed size
+     */
+    evictForSize(bytesNeeded: number): boolean;
+    /**
+     * Fairness monitoring - prevent one type from hogging cache
+     */
+    private startFairnessMonitor;
+    private checkFairness;
+    /**
+     * Force evict items of a specific type
+     */
+    private evictType;
+    /**
+     * Delete specific item from cache
+     */
+    delete(key: string): boolean;
+    /**
+     * Clear cache or specific type
+     */
+    clear(type?: 'hnsw' | 'metadata' | 'embedding' | 'other'): void;
+    /**
+     * Get cache statistics
+     */
+    getStats(): {
+        totalSize: number;
+        maxSize: number;
+        utilization: number;
+        itemCount: number;
+        typeSizes: {
+            hnsw: number;
+            metadata: number;
+            embedding: number;
+            other: number;
+        };
+        typeCounts: {
+            hnsw: number;
+            metadata: number;
+            embedding: number;
+            other: number;
+        };
+        typeAccessCounts: {
+            hnsw: number;
+            metadata: number;
+            embedding: number;
+            other: number;
+        };
+        totalAccessCount: number;
+        hitRate: number;
+    };
+    /**
+     * Save access patterns for cold start optimization
+     */
+    saveAccessPatterns(): Promise<any>;
+    /**
+     * Load access patterns for warm start
+     */
+    loadAccessPatterns(patterns: any): Promise<void>;
+}
+export declare function getGlobalCache(config?: UnifiedCacheConfig): UnifiedCache;
+export declare function clearGlobalCache(): void;

package/dist/utils/unifiedCache.js

@@ -0,0 +1,311 @@
+/**
+ * UnifiedCache - Single cache for both HNSW and MetadataIndex
+ * Prevents resource competition with cost-aware eviction
+ */
+import { prodLog } from './logger.js';
+export class UnifiedCache {
+    constructor(config = {}) {
+        this.cache = new Map();
+        this.access = new Map(); // Access counts
+        this.loadingPromises = new Map();
+        this.typeAccessCounts = { hnsw: 0, metadata: 0, embedding: 0, other: 0 };
+        this.totalAccessCount = 0;
+        this.currentSize = 0;
+        this.maxSize = config.maxSize || 2 * 1024 * 1024 * 1024; // 2GB default
+        this.config = {
+            enableRequestCoalescing: true,
+            enableFairnessCheck: true,
+            fairnessCheckInterval: 60000, // Check fairness every minute
+            persistPatterns: true,
+            ...config
+        };
+        if (this.config.enableFairnessCheck) {
+            this.startFairnessMonitor();
+        }
+    }
+    /**
+     * Get item from cache with request coalescing
+     */
+    async get(key, loadFn) {
+        // Update access tracking
+        this.access.set(key, (this.access.get(key) || 0) + 1);
+        this.totalAccessCount++;
+        // Check if in cache
+        const item = this.cache.get(key);
+        if (item) {
+            item.lastAccess = Date.now();
+            item.accessCount++;
+            this.typeAccessCounts[item.type]++;
+            return item.data;
+        }
+        // If no load function, return undefined
+        if (!loadFn) {
+            return undefined;
+        }
+        // Request coalescing - prevent stampede
+        if (this.config.enableRequestCoalescing && this.loadingPromises.has(key)) {
+            prodLog.debug('Request coalescing for key:', key);
+            return this.loadingPromises.get(key);
+        }
+        // Load data
+        const loadPromise = loadFn();
+        if (this.config.enableRequestCoalescing) {
+            this.loadingPromises.set(key, loadPromise);
+        }
+        try {
+            const data = await loadPromise;
+            return data;
+        }
+        finally {
+            if (this.config.enableRequestCoalescing) {
+                this.loadingPromises.delete(key);
+            }
+        }
+    }
+    /**
+     * Set item in cache with cost-aware eviction
+     */
+    set(key, data, type, size, rebuildCost = 1) {
+        // Make room if needed
+        while (this.currentSize + size > this.maxSize && this.cache.size > 0) {
+            this.evictLowestValue();
+        }
+        // Add to cache
+        const item = {
+            key,
+            type,
+            data,
+            size,
+            rebuildCost,
+            lastAccess: Date.now(),
+            accessCount: 1
+        };
+        // Update or add
+        const existing = this.cache.get(key);
+        if (existing) {
+            this.currentSize -= existing.size;
+        }
+        this.cache.set(key, item);
+        this.currentSize += size;
+        this.typeAccessCounts[type]++;
+        this.totalAccessCount++;
+    }
+    /**
+     * Evict item with lowest value (access count / rebuild cost)
+     */
+    evictLowestValue() {
+        let victim = null;
+        let lowestScore = Infinity;
+        for (const [key, item] of this.cache) {
+            // Calculate value score: access frequency / rebuild cost
+            const accessScore = (this.access.get(key) || 1);
+            const score = accessScore / Math.max(item.rebuildCost, 1);
+            if (score < lowestScore) {
+                lowestScore = score;
+                victim = key;
+            }
+        }
+        if (victim) {
+            const item = this.cache.get(victim);
+            prodLog.debug(`Evicting ${victim} (type: ${item.type}, score: ${lowestScore})`);
+            this.currentSize -= item.size;
+            this.cache.delete(victim);
+            // Keep access count for a while to prevent re-caching cold items
+            // this.access.delete(victim)  // Don't delete immediately
+        }
+    }
+    /**
+     * Size-aware eviction - try to match needed size
+     */
+    evictForSize(bytesNeeded) {
+        const candidates = [];
+        for (const [key, item] of this.cache) {
+            const score = (this.access.get(key) || 1) / item.rebuildCost;
+            candidates.push([key, score, item]);
+        }
+        // Sort by score (lower is worse)
+        candidates.sort((a, b) => a[1] - b[1]);
+        let freedBytes = 0;
+        const toEvict = [];
+        // Try to free exactly what we need
+        for (const [key, , item] of candidates) {
+            toEvict.push(key);
+            freedBytes += item.size;
+            if (freedBytes >= bytesNeeded) {
+                break;
+            }
+        }
+        // Evict selected items
+        for (const key of toEvict) {
+            const item = this.cache.get(key);
+            this.currentSize -= item.size;
+            this.cache.delete(key);
+        }
+        return freedBytes >= bytesNeeded;
+    }
+    /**
+     * Fairness monitoring - prevent one type from hogging cache
+     */
+    startFairnessMonitor() {
+        setInterval(() => {
+            this.checkFairness();
+        }, this.config.fairnessCheckInterval);
+    }
+    checkFairness() {
+        // Calculate type ratios in cache
+        const typeSizes = { hnsw: 0, metadata: 0, embedding: 0, other: 0 };
+        const typeCounts = { hnsw: 0, metadata: 0, embedding: 0, other: 0 };
+        for (const item of this.cache.values()) {
+            typeSizes[item.type] += item.size;
+            typeCounts[item.type]++;
+        }
+        // Calculate access ratios
+        const totalAccess = this.totalAccessCount || 1;
+        const accessRatios = {
+            hnsw: this.typeAccessCounts.hnsw / totalAccess,
+            metadata: this.typeAccessCounts.metadata / totalAccess,
+            embedding: this.typeAccessCounts.embedding / totalAccess,
+            other: this.typeAccessCounts.other / totalAccess
+        };
+        // Calculate size ratios
+        const totalSize = this.currentSize || 1;
+        const sizeRatios = {
+            hnsw: typeSizes.hnsw / totalSize,
+            metadata: typeSizes.metadata / totalSize,
+            embedding: typeSizes.embedding / totalSize,
+            other: typeSizes.other / totalSize
+        };
+        // Check for starvation (90% cache but <10% accesses)
+        for (const type of ['hnsw', 'metadata', 'embedding', 'other']) {
+            if (sizeRatios[type] > 0.9 && accessRatios[type] < 0.1) {
+                prodLog.warn(`Type ${type} is hogging cache (${(sizeRatios[type] * 100).toFixed(1)}% size, ${(accessRatios[type] * 100).toFixed(1)}% access)`);
+                this.evictType(type);
+            }
+        }
+    }
+    /**
+     * Force evict items of a specific type
+     */
+    evictType(type) {
+        const candidates = [];
+        for (const [key, item] of this.cache) {
+            if (item.type === type) {
+                const score = (this.access.get(key) || 1) / item.rebuildCost;
+                candidates.push([key, score, item]);
+            }
+        }
+        // Sort by score (lower is worse)
+        candidates.sort((a, b) => a[1] - b[1]);
+        // Evict bottom 20% of this type
+        const evictCount = Math.max(1, Math.floor(candidates.length * 0.2));
+        for (let i = 0; i < evictCount && i < candidates.length; i++) {
+            const [key, , item] = candidates[i];
+            this.currentSize -= item.size;
+            this.cache.delete(key);
+            prodLog.debug(`Fairness eviction: ${key} (type: ${type})`);
+        }
+    }
+    /**
+     * Delete specific item from cache
+     */
+    delete(key) {
+        const item = this.cache.get(key);
+        if (item) {
+            this.currentSize -= item.size;
+            this.cache.delete(key);
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Clear cache or specific type
+     */
+    clear(type) {
+        if (!type) {
+            this.cache.clear();
+            this.currentSize = 0;
+            return;
+        }
+        for (const [key, item] of this.cache) {
+            if (item.type === type) {
+                this.currentSize -= item.size;
+                this.cache.delete(key);
+            }
+        }
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        const typeSizes = { hnsw: 0, metadata: 0, embedding: 0, other: 0 };
+        const typeCounts = { hnsw: 0, metadata: 0, embedding: 0, other: 0 };
+        for (const item of this.cache.values()) {
+            typeSizes[item.type] += item.size;
+            typeCounts[item.type]++;
+        }
+        return {
+            totalSize: this.currentSize,
+            maxSize: this.maxSize,
+            utilization: this.currentSize / this.maxSize,
+            itemCount: this.cache.size,
+            typeSizes,
+            typeCounts,
+            typeAccessCounts: this.typeAccessCounts,
+            totalAccessCount: this.totalAccessCount,
+            hitRate: this.cache.size > 0 ?
+                Array.from(this.cache.values()).reduce((sum, item) => sum + item.accessCount, 0) / this.totalAccessCount : 0
+        };
+    }
+    /**
+     * Save access patterns for cold start optimization
+     */
+    async saveAccessPatterns() {
+        if (!this.config.persistPatterns)
+            return;
+        const patterns = Array.from(this.cache.entries())
+            .map(([key, item]) => ({
+            key,
+            type: item.type,
+            accessCount: this.access.get(key) || 0,
+            size: item.size,
+            rebuildCost: item.rebuildCost
+        }))
+            .sort((a, b) => b.accessCount - a.accessCount);
+        return {
+            patterns,
+            typeAccessCounts: this.typeAccessCounts,
+            timestamp: Date.now()
+        };
+    }
+    /**
+     * Load access patterns for warm start
+     */
+    async loadAccessPatterns(patterns) {
+        if (!patterns?.patterns)
+            return;
+        // Pre-populate access counts
+        for (const pattern of patterns.patterns) {
+            this.access.set(pattern.key, pattern.accessCount);
+        }
+        // Restore type access counts
+        if (patterns.typeAccessCounts) {
+            this.typeAccessCounts = patterns.typeAccessCounts;
+        }
+        prodLog.debug('Loaded access patterns:', patterns.patterns.length, 'items');
+    }
+}
+// Export singleton for global coordination
+let globalCache = null;
+export function getGlobalCache(config) {
+    if (!globalCache) {
+        globalCache = new UnifiedCache(config);
+    }
+    return globalCache;
+}
+export function clearGlobalCache() {
+    if (globalCache) {
+        globalCache.clear();
+        globalCache = null;
+    }
+}
+//# sourceMappingURL=unifiedCache.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "1.5.0",
-  "description": "Multi-Dimensional AI Database - Vector similarity, graph relationships, metadata facets with HNSW indexing and OPFS storage",
+  "version": "2.0.0",
+  "description": "Multi-Dimensional AI Database - Vector search, graph relationships, field filtering with Triple Intelligence Engine, HNSW indexing and universal storage",
   "main": "dist/index.js",
   "module": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -45,14 +45,6 @@
       "import": "./dist/utils/textEncoding.js",
       "types": "./dist/utils/textEncoding.d.ts"
     },
-    "./dist/utils/textEncoding.js": {
-      "import": "./dist/utils/textEncoding.js",
-      "types": "./dist/utils/textEncoding.d.ts"
-    },
-    "./dist/setup.js": {
-      "import": "./dist/setup.js",
-      "types": "./dist/setup.d.ts"
-    },
     "./browserFramework": {
       "import": "./dist/browserFramework.js",
       "types": "./dist/browserFramework.d.ts"
@@ -60,172 +52,95 @@
     "./universal": {
       "import": "./dist/universal/index.js",
       "types": "./dist/universal/index.d.ts"
-    },
-    "./universal/uuid": {
-      "import": "./dist/universal/uuid.js",
-      "types": "./dist/universal/uuid.d.ts"
-    },
-    "./universal/crypto": {
-      "import": "./dist/universal/crypto.js",
-      "types": "./dist/universal/crypto.d.ts"
-    },
-    "./universal/fs": {
-      "import": "./dist/universal/fs.js",
-      "types": "./dist/universal/fs.d.ts"
-    },
-    "./universal/path": {
-      "import": "./dist/universal/path.js",
-      "types": "./dist/universal/path.d.ts"
-    },
-    "./universal/events": {
-      "import": "./dist/universal/events.js",
-      "types": "./dist/universal/events.d.ts"
     }
   },
   "engines": {
-    "node": ">=24.4.0"
+    "node": ">=18.0.0"
   },
   "scripts": {
-    "prebuild": "echo 'Prebuild step - no version generation needed'",
-    "build": "tsc",
-    "build:browser": "npm run build && vite build --config vite.browser.config.ts",
-    "build:framework": "tsc",
-    "start": "node dist/framework.js",
+    "build": "npm run build:patterns && tsc",
+    "build:patterns": "tsx scripts/buildEmbeddedPatterns.ts",
     "prepare": "npm run build",
-    "test": "vitest run",
-    "test:memory": "node --max-old-space-size=4096 --expose-gc ./node_modules/vitest/vitest.mjs run",
-    "test:watch": "vitest",
-    "test:ui": "vitest --ui",
-    "test:fast": "vitest run --reporter=dot --silent",
-    "test:node": "vitest run tests/environment.node.test.ts tests/core.test.ts tests/vector-operations.test.ts tests/tensorflow-patch.test.ts",
-    "test:browser": "vitest run tests/environment.browser.test.ts --environment jsdom",
-    "test:core": "vitest run tests/core.test.ts",
-    "test:coverage": "vitest run --coverage",
-    "test:size": "vitest run tests/package-size-limit.test.ts",
-    "test:all": "npm run build && vitest run",
-    "test:report:json": "vitest run --reporter json",
-    "test:error-handling": "vitest run tests/error-handling.test.ts",
-    "test:edge-cases": "vitest run tests/edge-cases.test.ts",
-    "test:storage": "vitest run tests/storage-adapter-coverage.test.ts",
-    "broadcast:server": "npm run build && node dist/scripts/start-broadcast-server.js",
-    "broadcast:local": "npm run build && node dist/scripts/start-broadcast-server.js",
-    "broadcast:cloud": "npm run build && node dist/scripts/start-broadcast-server.js --cloud",
-    "claude:jarvis": "npm run build && node dist/scripts/claude-jarvis.js",
-    "claude:picasso": "npm run build && node dist/scripts/claude-picasso.js",
-    "test:environments": "vitest run tests/multi-environment.test.ts",
-    "test:specialized": "vitest run tests/specialized-scenarios.test.ts",
-    "test:performance": "vitest run tests/performance.test.ts",
-    "test:install": "vitest run tests/package-install.test.ts",
-    "test:docker": "vitest run tests/custom-models-path.test.ts",
-    "test:extraction": "node tests/auto-extraction.test.js",
-    "extract-models": "node scripts/extract-models.js",
-    "test:comprehensive": "npm run test:error-handling && npm run test:edge-cases && npm run test:storage && npm run test:environments && npm run test:specialized",
-    "test:release": "npm run build && vitest run tests/release-validation.test.ts tests/regression.test.ts tests/unified-api.test.ts tests/core.test.ts --reporter=verbose",
-    "test:1.0": "vitest run tests/unified-api.test.ts tests/cli.test.ts --reporter=verbose",
-    "_generate-pdf": "node dev/scripts/generate-architecture-pdf.js",
-    "_release": "standard-version",
-    "_release:patch": "standard-version --release-as patch",
-    "_release:minor": "standard-version --release-as minor",
-    "_release:major": "standard-version --release-as major",
-    "_release:dry-run": "standard-version --dry-run",
-    "_github-release": "node scripts/create-github-release.js",
-    "_changelog:check": "echo 'Changelog is now automatically generated from commit messages'",
-    "_lint": "eslint --ext .ts,.js src/",
-    "_lint:fix": "eslint --ext .ts,.js src/ --fix",
-    "_format": "prettier --write \"src/**/*.{ts,js}\"",
-    "_check:format": "prettier --check \"src/**/*.{ts,js}\"",
-    "_check:style": "node scripts/check-code-style.js",
-    "_deploy": "npm run build && npm publish",
-    "_workflow": "node scripts/release-workflow.js",
-    "_workflow:patch": "node scripts/release-workflow.js patch",
-    "_workflow:minor": "node scripts/release-workflow.js minor",
-    "_workflow:major": "node scripts/release-workflow.js major",
-    "_workflow:dry-run": "npm run build && npm test && npm run _release:dry-run",
-    "_dry-run": "npm pack --dry-run",
+    "test": "npm run test:unit",
+    "test:watch": "vitest --config tests/configs/vitest.unit.config.ts",
+    "test:coverage": "vitest run --config tests/configs/vitest.unit.config.ts --coverage",
+    "test:unit": "vitest run --config tests/configs/vitest.unit.config.ts",
+    "test:integration": "NODE_OPTIONS='--max-old-space-size=32768' vitest run --config tests/configs/vitest.integration.config.ts",
+    "test:all": "npm run test:unit && npm run test:integration",
+    "test:ci-unit": "CI=true vitest run --config tests/configs/vitest.unit.config.ts",
+    "test:ci-integration": "NODE_OPTIONS='--max-old-space-size=16384' CI=true vitest run --config tests/configs/vitest.integration.config.ts",
+    "test:ci": "npm run test:ci-unit",
     "download-models": "node scripts/download-models.cjs",
-    "prepare-models": "node scripts/prepare-models.js",
     "models:verify": "node scripts/ensure-models.js",
-    "models:download": "BRAINY_ALLOW_REMOTE_MODELS=true node scripts/download-models.cjs"
+    "lint": "eslint --ext .ts,.js src/",
+    "lint:fix": "eslint --ext .ts,.js src/ --fix",
+    "format": "prettier --write \"src/**/*.{ts,js}\"",
+    "format:check": "prettier --check \"src/**/*.{ts,js}\""
   },
   "keywords": [
+    "ai-database",
     "vector-database",
+    "graph-database",
+    "field-filtering",
+    "triple-intelligence",
     "hnsw",
-    "opfs",
-    "origin-private-file-system",
     "embeddings",
-    "graph-database",
-    "streaming-data"
+    "semantic-search",
+    "machine-learning",
+    "artificial-intelligence",
+    "data-storage",
+    "indexing",
+    "typescript"
   ],
-  "author": "David Snelling (david@soulcraft.com)",
+  "author": "Brainy Contributors",
   "license": "MIT",
   "private": false,
   "publishConfig": {
     "access": "public"
   },
-  "homepage": "https://github.com/soulcraftlabs/brainy",
+  "homepage": "https://github.com/brainy-org/brainy",
   "bugs": {
-    "url": "https://github.com/soulcraftlabs/brainy/issues"
+    "url": "https://github.com/brainy-org/brainy/issues"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/soulcraftlabs/brainy.git"
+    "url": "git+https://github.com/brainy-org/brainy.git"
   },
   "files": [
     "dist/**/*.js",
     "dist/**/*.d.ts",
-    "!dist/framework.js",
-    "!dist/framework.js.map",
-    "!dist/framework.min.js",
-    "!dist/framework.min.js.map",
+    "bin/",
+    "scripts/download-models.cjs",
+    "scripts/ensure-models.js",
+    "scripts/prepare-models.js",
+    "brainy.png",
     "LICENSE",
     "README.md",
-    "brainy.png",
-    "scripts/download-models.cjs",
-    "OFFLINE_MODELS.md"
+    "CHANGELOG.md"
   ],
   "devDependencies": {
     "@rollup/plugin-commonjs": "^28.0.6",
     "@rollup/plugin-node-resolve": "^16.0.1",
     "@rollup/plugin-replace": "^6.0.2",
     "@rollup/plugin-terser": "^0.4.4",
-    "@types/express": "^5.0.3",
-    "@types/jsdom": "^21.1.7",
     "@types/node": "^20.11.30",
-    "@types/prompts": "^2.4.9",
     "@types/uuid": "^10.0.0",
-    "@types/ws": "^8.18.1",
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "@typescript-eslint/parser": "^8.0.0",
-    "@vitejs/plugin-basic-ssl": "^2.1.0",
     "@vitest/coverage-v8": "^3.2.4",
-    "@vitest/ui": "^3.2.4",
-    "aws-sdk-client-mock": "^4.1.0",
-    "aws-sdk-client-mock-jest": "^4.1.0",
-    "buffer": "^6.0.3",
-    "crypto-browserify": "^3.12.1",
-    "eslint": "^9.0.0",
-    "express": "^5.1.0",
-    "happy-dom": "^18.0.1",
-    "jsdom": "^26.1.0",
-    "process": "^0.11.10",
-    "puppeteer": "^22.15.0",
-    "standard-version": "^9.5.0",
-    "tslib": "^2.6.2",
+    "tsx": "^4.19.2",
     "typescript": "^5.4.5",
-    "vite": "^7.1.1",
     "vitest": "^3.2.4"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.540.0",
     "@huggingface/transformers": "^3.1.0",
-    "@smithy/node-http-handler": "^4.1.1",
-    "boxen": "^7.1.1",
+    "boxen": "^8.0.1",
     "chalk": "^5.3.0",
-    "cli-table3": "^0.6.3",
+    "cli-table3": "^0.6.5",
     "commander": "^11.1.0",
-    "dotenv": "^16.4.5",
-    "inquirer": "^12.9.1",
-    "ora": "^8.0.1",
+    "inquirer": "^12.9.3",
+    "ora": "^8.2.0",
     "prompts": "^2.4.2",
     "uuid": "^9.0.1"
   },