@soulcraft/brainy 1.5.0 → 2.0.1

package/dist/utils/embedding.js

@@ -6,6 +6,15 @@ import { isBrowser } from './environment.js';
 import { ModelManager } from '../embeddings/model-manager.js';
 // @ts-ignore - Transformers.js is now the primary embedding library
 import { pipeline, env } from '@huggingface/transformers';
+// CRITICAL: Disable ONNX memory arena to prevent 4-8GB allocation
+// This is needed for BOTH production and testing - reduces memory by 50-75%
+if (typeof process !== 'undefined' && process.env) {
+    process.env.ORT_DISABLE_MEMORY_ARENA = '1';
+    process.env.ORT_DISABLE_MEMORY_PATTERN = '1';
+    // Also limit ONNX thread count for more predictable memory usage
+    process.env.ORT_INTRA_OP_NUM_THREADS = '2';
+    process.env.ORT_INTER_OP_NUM_THREADS = '2';
+}
 /**
  * Detect the best available GPU device for the current environment
  */
@@ -89,7 +98,7 @@ export class TransformerEmbedding {
             verbose: this.verbose,
             cacheDir: options.cacheDir || './models',
             localFilesOnly: localFilesOnly,
-            dtype: options.dtype || 'fp32',
+            dtype: options.dtype || 'q8', // Changed from fp32 to q8 for 75% memory reduction
             device: options.device || 'auto'
         };
         if (this.verbose) {
@@ -203,11 +212,19 @@ export class TransformerEmbedding {
                 : this.options.cacheDir;
             this.logger('log', `Loading Transformer model: ${this.options.model} on device: ${device}`);
             const startTime = Date.now();
-            // Load the feature extraction pipeline with GPU support
+            // Load the feature extraction pipeline with memory optimizations
             const pipelineOptions = {
                 cache_dir: cacheDir,
                 local_files_only: isBrowser() ? false : this.options.localFilesOnly,
-                dtype: this.options.dtype
+                dtype: this.options.dtype || 'q8', // Use quantized model for lower memory
+                // CRITICAL: ONNX memory optimizations
+                session_options: {
+                    enableCpuMemArena: false, // Disable pre-allocated memory arena
+                    enableMemPattern: false, // Disable memory pattern optimization
+                    interOpNumThreads: 2, // Limit thread count
+                    intraOpNumThreads: 2, // Limit parallelism
+                    graphOptimizationLevel: 'all'
+                }
             };
             // Add device configuration for GPU acceleration
             if (device !== 'cpu') {

package/dist/utils/hybridModelManager.js

@@ -75,6 +75,11 @@ class HybridModelManager {
                 process.env.FUNCTIONS_WORKER_RUNTIME);
             const isDocker = typeof process !== 'undefined' && (process.env.DOCKER_CONTAINER ||
                 process.env.KUBERNETES_SERVICE_HOST);
+            // Respect BRAINY_ALLOW_REMOTE_MODELS environment variable first
+            let forceLocalOnly = false;
+            if (process.env.BRAINY_ALLOW_REMOTE_MODELS !== undefined) {
+                forceLocalOnly = process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true';
+            }
             // Smart configuration based on environment
             let options = {
                 verbose: !isTest && !isServerless,
@@ -85,7 +90,7 @@ class HybridModelManager {
             if (isBrowser) {
                 options = {
                     ...options,
-                    localFilesOnly: false,
+                    localFilesOnly: forceLocalOnly || false, // Respect environment variable
                     dtype: 'q8',
                     device: 'cpu',
                     verbose: false
@@ -94,7 +99,7 @@ class HybridModelManager {
             else if (isServerless) {
                 options = {
                     ...options,
-                    localFilesOnly: true,
+                    localFilesOnly: forceLocalOnly || true, // Default true for serverless, but respect env
                     dtype: 'q8',
                     device: 'cpu',
                     verbose: false
@@ -103,7 +108,7 @@ class HybridModelManager {
             else if (isDocker) {
                 options = {
                     ...options,
-                    localFilesOnly: true,
+                    localFilesOnly: forceLocalOnly || true, // Default true for docker, but respect env
                     dtype: 'fp32',
                     device: 'auto',
                     verbose: false
@@ -113,7 +118,7 @@ class HybridModelManager {
                 // CRITICAL FOR TESTS: Allow remote downloads but be smart about it
                 options = {
                     ...options,
-                    localFilesOnly: false,
+                    localFilesOnly: forceLocalOnly || false, // Respect environment variable for tests
                     dtype: 'q8',
                     device: 'cpu',
                     verbose: false
@@ -122,7 +127,7 @@ class HybridModelManager {
             else {
                 options = {
                     ...options,
-                    localFilesOnly: false,
+                    localFilesOnly: forceLocalOnly || false, // Respect environment variable for default node
                     dtype: 'q8',
                     device: 'auto',
                     verbose: true

package/dist/utils/metadataFilter.d.ts

@@ -5,31 +5,45 @@
  */
 import { SearchResult, HNSWNoun } from '../coreTypes.js';
 /**
- * MongoDB-style query operators
+ * Brainy Field Operators (BFO) - Our own field query system
+ * Designed for performance, clarity, and patent independence
  */
-export interface QueryOperators {
-    $eq?: any;
-    $ne?: any;
-    $gt?: any;
-    $gte?: any;
-    $lt?: any;
-    $lte?: any;
-    $in?: any[];
-    $nin?: any[];
-    $exists?: boolean;
-    $regex?: string | RegExp;
-    $includes?: any;
-    $all?: any[];
-    $size?: number;
-    $and?: MetadataFilter[];
-    $or?: MetadataFilter[];
-    $not?: MetadataFilter;
+export interface BrainyFieldOperators {
+    equals?: any;
+    notEquals?: any;
+    is?: any;
+    isNot?: any;
+    greaterThan?: any;
+    greaterEqual?: any;
+    lessThan?: any;
+    lessEqual?: any;
+    between?: [any, any];
+    oneOf?: any[];
+    noneOf?: any[];
+    contains?: any;
+    excludes?: any;
+    hasAll?: any[];
+    length?: number;
+    exists?: boolean;
+    missing?: boolean;
+    matches?: string | RegExp;
+    startsWith?: string;
+    endsWith?: string;
+    allOf?: MetadataFilter[];
+    anyOf?: MetadataFilter[];
+    not?: MetadataFilter;
+    eq?: any;
+    ne?: any;
+    gt?: any;
+    gte?: any;
+    lt?: any;
+    lte?: any;
 }
 /**
  * Metadata filter definition
  */
 export interface MetadataFilter {
-    [key: string]: any | QueryOperators;
+    [key: string]: any | BrainyFieldOperators;
 }
 /**
  * Options for metadata filtering

package/dist/utils/metadataFilter.js

@@ -11,55 +11,67 @@ function matchesQuery(value, query) {
     if (typeof query !== 'object' || query === null || Array.isArray(query)) {
         return value === query;
     }
-    // Check for MongoDB-style operators
+    // Check for Brainy Field Operators (BFO)
     for (const [op, operand] of Object.entries(query)) {
         switch (op) {
-            case '$eq':
+            // Equality operators
+            case 'equals':
+            case 'is':
+            case 'eq':
                 if (value !== operand)
                     return false;
                 break;
-            case '$ne':
+            case 'notEquals':
+            case 'isNot':
+            case 'ne':
                 if (value === operand)
                     return false;
                 break;
-            case '$gt':
+            // Comparison operators
+            case 'greaterThan':
+            case 'gt':
                 if (typeof value !== 'number' || typeof operand !== 'number' || !(value > operand))
                     return false;
                 break;
-            case '$gte':
+            case 'greaterEqual':
+            case 'gte':
                 if (typeof value !== 'number' || typeof operand !== 'number' || !(value >= operand))
                     return false;
                 break;
-            case '$lt':
+            case 'lessThan':
+            case 'lt':
                 if (typeof value !== 'number' || typeof operand !== 'number' || !(value < operand))
                     return false;
                 break;
-            case '$lte':
+            case 'lessEqual':
+            case 'lte':
                 if (typeof value !== 'number' || typeof operand !== 'number' || !(value <= operand))
                     return false;
                 break;
-            case '$in':
-                if (!Array.isArray(operand) || !operand.includes(value))
+            case 'between':
+                if (!Array.isArray(operand) || operand.length !== 2)
                     return false;
-                break;
-            case '$nin':
-                if (!Array.isArray(operand) || operand.includes(value))
+                if (typeof value !== 'number' || !(value >= operand[0] && value <= operand[1]))
                     return false;
                 break;
-            case '$exists':
-                if ((value !== undefined) !== operand)
+            // Array/Set operators
+            case 'oneOf':
+                if (!Array.isArray(operand) || !operand.includes(value))
                     return false;
                 break;
-            case '$regex':
-                const regex = typeof operand === 'string' ? new RegExp(operand) : operand;
-                if (!(regex instanceof RegExp) || !regex.test(String(value)))
+            case 'noneOf':
+                if (!Array.isArray(operand) || operand.includes(value))
                     return false;
                 break;
-            case '$includes':
+            case 'contains':
                 if (!Array.isArray(value) || !value.includes(operand))
                     return false;
                 break;
-            case '$all':
+            case 'excludes':
+                if (!Array.isArray(value) || value.includes(operand))
+                    return false;
+                break;
+            case 'hasAll':
                 if (!Array.isArray(value) || !Array.isArray(operand))
                     return false;
                 for (const item of operand) {
@@ -67,10 +79,33 @@ function matchesQuery(value, query) {
                         return false;
                 }
                 break;
-            case '$size':
+            case 'length':
                 if (!Array.isArray(value) || value.length !== operand)
                     return false;
                 break;
+            // Existence operators
+            case 'exists':
+                if ((value !== undefined) !== operand)
+                    return false;
+                break;
+            case 'missing':
+                if ((value === undefined) !== operand)
+                    return false;
+                break;
+            // Pattern operators
+            case 'matches':
+                const regex = typeof operand === 'string' ? new RegExp(operand) : operand;
+                if (!(regex instanceof RegExp) || !regex.test(String(value)))
+                    return false;
+                break;
+            case 'startsWith':
+                if (typeof value !== 'string' || !value.startsWith(String(operand)))
+                    return false;
+                break;
+            case 'endsWith':
+                if (typeof value !== 'string' || !value.endsWith(String(operand)))
+                    return false;
+                break;
             default:
                 // Unknown operator, treat as field name
                 if (!matchesFieldQuery(value, op, operand))
@@ -109,7 +144,7 @@ export function matchesMetadataFilter(metadata, filter) {
     }
     for (const [key, query] of Object.entries(filter)) {
         // Handle logical operators
-        if (key === '$and') {
+        if (key === 'allOf') {
             if (!Array.isArray(query))
                 return false;
             for (const subFilter of query) {
@@ -118,7 +153,7 @@ export function matchesMetadataFilter(metadata, filter) {
             }
             continue;
         }
-        if (key === '$or') {
+        if (key === 'anyOf') {
             if (!Array.isArray(query))
                 return false;
             let matched = false;
@@ -132,7 +167,7 @@ export function matchesMetadataFilter(metadata, filter) {
                 return false;
             continue;
         }
-        if (key === '$not') {
+        if (key === 'not') {
             if (matchesMetadataFilter(metadata, query))
                 return false;
             continue;

package/dist/utils/metadataIndex.d.ts

@@ -28,10 +28,6 @@ export interface MetadataIndexConfig {
     indexedFields?: string[];
     excludeFields?: string[];
 }
-/**
- * Manages metadata indexes for fast filtering
- * Maintains inverted indexes: field+value -> list of IDs
- */
 export declare class MetadataIndexManager {
     private storage;
     private config;
@@ -43,11 +39,34 @@ export declare class MetadataIndexManager {
     private dirtyFields;
     private lastFlushTime;
     private autoFlushThreshold;
+    private sortedIndices;
+    private numericFields;
+    private unifiedCache;
     constructor(storage: StorageAdapter, config?: MetadataIndexConfig);
     /**
      * Get index key for field and value
      */
     private getIndexKey;
+    /**
+     * Ensure sorted index exists for a field (for range queries)
+     */
+    private ensureSortedIndex;
+    /**
+     * Build sorted index for a field from hash index
+     */
+    private buildSortedIndex;
+    /**
+     * Binary search for range start (inclusive or exclusive)
+     */
+    private binarySearchStart;
+    /**
+     * Binary search for range end (inclusive or exclusive)
+     */
+    private binarySearchEnd;
+    /**
+     * Get IDs matching a range query
+     */
+    private getIdsForRange;
     /**
      * Generate field index filename for filter discovery
      */
@@ -101,13 +120,17 @@ export declare class MetadataIndexManager {
      */
     getFilterFields(): Promise<string[]>;
     /**
-     * Convert MongoDB-style filter to simple field-value criteria for indexing
+     * Convert Brainy Field Operator filter to simple field-value criteria for indexing
      */
     private convertFilterToCriteria;
     /**
-     * Get IDs matching MongoDB-style metadata filter using indexes where possible
+     * Get IDs matching Brainy Field Operator metadata filter using indexes where possible
      */
     getIdsForFilter(filter: any): Promise<string[]>;
+    /**
+     * DEPRECATED - Old implementation for backward compatibility
+     */
+    private getIdsForFilterOld;
     /**
      * Get IDs matching multiple criteria (intersection) - LEGACY METHOD
      * @deprecated Use getIdsForFilter instead
@@ -130,6 +153,14 @@ export declare class MetadataIndexManager {
      * Save field index to storage
      */
     private saveFieldIndex;
+    /**
+     * Save sorted index to storage for range queries
+     */
+    private saveSortedIndex;
+    /**
+     * Load sorted index from storage
+     */
+    private loadSortedIndex;
     /**
      * Get index statistics
      */