@soulcraft/brainy 5.5.0 → 5.6.0

package/dist/storage/baseStorage.js

@@ -77,12 +77,14 @@ export class BaseStorage extends BaseStorageAdapter {
         this.cowEnabled = false;
         // Type-first indexing support (v5.4.0)
         // Built into all storage adapters for billion-scale efficiency
-        this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT); // 124 bytes
-        this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT); // 160 bytes
-        // Total: 284 bytes (99.76% reduction vs Map-based tracking)
+        this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT); // 168 bytes (Stage 3: 42 types)
+        this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT); // 508 bytes (Stage 3: 127 types)
+        // Total: 676 bytes (99.2% reduction vs Map-based tracking)
         // Type cache for O(1) lookups after first access
         this.nounTypeCache = new Map();
         this.verbTypeCache = new Map();
+        // v5.5.0: Track if type counts have been rebuilt (prevent repeated rebuilds)
+        this.typeCountsRebuilt = false;
     }
     /**
      * Analyze a storage key to determine its routing and path
@@ -825,14 +827,30 @@ export class BaseStorage extends BaseStorageAdapter {
      * Get nouns with pagination (v5.4.0: Type-first implementation)
      *
      * CRITICAL: This method is required for brain.find() to work!
-     * Iterates through all noun types to find entities.
+     * Iterates through noun types with billion-scale optimizations.
+     *
+     * ARCHITECTURE: Reads storage directly (not indexes) to avoid circular dependencies.
+     * Storage → Indexes (one direction only). GraphAdjacencyIndex built FROM storage.
+     *
+     * OPTIMIZATIONS (v5.5.0):
+     * - Skip empty types using nounCountsByType[] tracking (O(1) check)
+     * - Early termination when offset + limit entities collected
+     * - Memory efficient: Never loads full dataset
      */
     async getNounsWithPagination(options) {
         await this.ensureInitialized();
-        const { limit, offset, filter } = options;
-        const allNouns = [];
-        // v5.4.0: Iterate through all noun types (type-first architecture)
-        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+        const { limit, offset = 0, filter } = options;
+        const collectedNouns = [];
+        const targetCount = offset + limit; // Early termination target
+        // v5.5.0 BUG FIX: Only use optimization if counts are reliable
+        const totalNounCountFromArray = this.nounCountsByType.reduce((sum, c) => sum + c, 0);
+        const useOptimization = totalNounCountFromArray > 0;
+        // v5.5.0: Iterate through noun types with billion-scale optimizations
+        for (let i = 0; i < NOUN_TYPE_COUNT && collectedNouns.length < targetCount; i++) {
+            // OPTIMIZATION 1: Skip empty types (only if counts are reliable)
+            if (useOptimization && this.nounCountsByType[i] === 0) {
+                continue;
+            }
             const type = TypeUtils.getNounFromIndex(i);
             // If filtering by type, skip other types
             if (filter?.nounType) {
@@ -846,6 +864,10 @@ export class BaseStorage extends BaseStorageAdapter {
                 // List all noun files for this type
                 const nounFiles = await this.listObjectsInBranch(typeDir);
                 for (const nounPath of nounFiles) {
+                    // OPTIMIZATION 2: Early termination (stop when we have enough)
+                    if (collectedNouns.length >= targetCount) {
+                        break;
+                    }
                     // Skip if not a .json file
                     if (!nounPath.endsWith('.json'))
                         continue;
@@ -864,7 +886,7 @@ export class BaseStorage extends BaseStorageAdapter {
                                     }
                                 }
                                 // Combine noun + metadata (v5.4.0: Extract standard fields to top-level)
-                                allNouns.push({
+                                collectedNouns.push({
                                     ...noun,
                                     type: metadata.noun || type, // Required: Extract type from metadata
                                     confidence: metadata.confidence,
@@ -892,19 +914,104 @@ export class BaseStorage extends BaseStorageAdapter {
                 // Skip types that have no data
             }
         }
-        // Apply pagination
-        const totalCount = allNouns.length;
-        const paginatedNouns = allNouns.slice(offset, offset + limit);
-        const hasMore = offset + limit < totalCount;
+        // Apply pagination (v5.5.0: Efficient slicing after early termination)
+        const paginatedNouns = collectedNouns.slice(offset, offset + limit);
+        const hasMore = collectedNouns.length >= targetCount;
         return {
             items: paginatedNouns,
-            totalCount,
+            totalCount: collectedNouns.length, // Accurate count of collected results
             hasMore,
             nextCursor: hasMore && paginatedNouns.length > 0
                 ? paginatedNouns[paginatedNouns.length - 1].id
                 : undefined
         };
     }
+    /**
+     * Get verbs with pagination (v5.5.0: Type-first implementation with billion-scale optimizations)
+     *
+     * CRITICAL: This method is required for brain.getRelations() to work!
+     * Iterates through verb types with the same optimizations as nouns.
+     *
+     * ARCHITECTURE: Reads storage directly (not indexes) to avoid circular dependencies.
+     * Storage → Indexes (one direction only). GraphAdjacencyIndex built FROM storage.
+     *
+     * OPTIMIZATIONS (v5.5.0):
+     * - Skip empty types using verbCountsByType[] tracking (O(1) check)
+     * - Early termination when offset + limit verbs collected
+     * - Memory efficient: Never loads full dataset
+     * - Inline filtering for sourceId, targetId, verbType
+     */
+    async getVerbsWithPagination(options) {
+        await this.ensureInitialized();
+        const { limit, offset = 0, filter } = options;
+        const collectedVerbs = [];
+        const targetCount = offset + limit; // Early termination target
+        // v5.5.0 BUG FIX: Only use optimization if counts are reliable
+        const totalVerbCountFromArray = this.verbCountsByType.reduce((sum, c) => sum + c, 0);
+        const useOptimization = totalVerbCountFromArray > 0;
+        // v5.5.0: Iterate through verb types with billion-scale optimizations
+        for (let i = 0; i < VERB_TYPE_COUNT && collectedVerbs.length < targetCount; i++) {
+            // OPTIMIZATION 1: Skip empty types (only if counts are reliable)
+            if (useOptimization && this.verbCountsByType[i] === 0) {
+                continue;
+            }
+            const type = TypeUtils.getVerbFromIndex(i);
+            // If filtering by verbType, skip other types
+            if (filter?.verbType) {
+                const filterTypes = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
+                if (!filterTypes.includes(type)) {
+                    continue;
+                }
+            }
+            try {
+                const verbsOfType = await this.getVerbsByType_internal(type);
+                // Apply filtering inline (memory efficient)
+                for (const verb of verbsOfType) {
+                    // OPTIMIZATION 2: Early termination (stop when we have enough)
+                    if (collectedVerbs.length >= targetCount) {
+                        break;
+                    }
+                    // Apply filters if specified
+                    if (filter) {
+                        // Filter by sourceId
+                        if (filter.sourceId) {
+                            const sourceIds = Array.isArray(filter.sourceId)
+                                ? filter.sourceId
+                                : [filter.sourceId];
+                            if (!sourceIds.includes(verb.sourceId)) {
+                                continue;
+                            }
+                        }
+                        // Filter by targetId
+                        if (filter.targetId) {
+                            const targetIds = Array.isArray(filter.targetId)
+                                ? filter.targetId
+                                : [filter.targetId];
+                            if (!targetIds.includes(verb.targetId)) {
+                                continue;
+                            }
+                        }
+                    }
+                    // Verb passed all filters - add to collection
+                    collectedVerbs.push(verb);
+                }
+            }
+            catch (error) {
+                // Skip types that have no data (directory may not exist)
+            }
+        }
+        // Apply pagination (v5.5.0: Efficient slicing after early termination)
+        const paginatedVerbs = collectedVerbs.slice(offset, offset + limit);
+        const hasMore = collectedVerbs.length >= targetCount;
+        return {
+            items: paginatedVerbs,
+            totalCount: collectedVerbs.length, // Accurate count of collected results
+            hasMore,
+            nextCursor: hasMore && paginatedVerbs.length > 0
+                ? paginatedVerbs[paginatedVerbs.length - 1].id
+                : undefined
+        };
+    }
     /**
      * Get verbs with pagination and filtering
      * @param options Pagination and filtering options
@@ -1080,12 +1187,84 @@ export class BaseStorage extends BaseStorageAdapter {
                     nextCursor: result.nextCursor
                 };
             }
-            // Storage adapter does not support pagination
-            console.error('Storage adapter does not support pagination. The deprecated getAllVerbs_internal() method has been removed. Please implement getVerbsWithPagination() in your storage adapter.');
+            // UNIVERSAL FALLBACK: Iterate through verb types with early termination (billion-scale safe)
+            // This approach works for ALL storage adapters without requiring adapter-specific pagination
+            console.warn('Using universal type-iteration strategy for getVerbs(). ' +
+                'This works for all adapters but may be slower than native pagination. ' +
+                'For optimal performance at scale, storage adapters can implement getVerbsWithPagination().');
+            const collectedVerbs = [];
+            let totalScanned = 0;
+            const targetCount = offset + limit; // We need this many verbs total (including offset)
+            // v5.5.0 BUG FIX: Check if optimization should be used
+            // Only use type-skipping optimization if counts are non-zero (reliable)
+            const totalVerbCountFromArray = this.verbCountsByType.reduce((sum, c) => sum + c, 0);
+            const useOptimization = totalVerbCountFromArray > 0;
+            // Iterate through all 127 verb types (Stage 3 CANONICAL) with early termination
+            // OPTIMIZATION: Skip types with zero count (only if counts are reliable)
+            for (let i = 0; i < VERB_TYPE_COUNT && collectedVerbs.length < targetCount; i++) {
+                // Skip empty types for performance (but only if optimization is enabled)
+                if (useOptimization && this.verbCountsByType[i] === 0) {
+                    continue;
+                }
+                const type = TypeUtils.getVerbFromIndex(i);
+                try {
+                    const verbsOfType = await this.getVerbsByType_internal(type);
+                    // Apply filtering inline (memory efficient)
+                    for (const verb of verbsOfType) {
+                        // Apply filters if specified
+                        if (options?.filter) {
+                            // Filter by sourceId
+                            if (options.filter.sourceId) {
+                                const sourceIds = Array.isArray(options.filter.sourceId)
+                                    ? options.filter.sourceId
+                                    : [options.filter.sourceId];
+                                if (!sourceIds.includes(verb.sourceId)) {
+                                    continue;
+                                }
+                            }
+                            // Filter by targetId
+                            if (options.filter.targetId) {
+                                const targetIds = Array.isArray(options.filter.targetId)
+                                    ? options.filter.targetId
+                                    : [options.filter.targetId];
+                                if (!targetIds.includes(verb.targetId)) {
+                                    continue;
+                                }
+                            }
+                            // Filter by verbType
+                            if (options.filter.verbType) {
+                                const verbTypes = Array.isArray(options.filter.verbType)
+                                    ? options.filter.verbType
+                                    : [options.filter.verbType];
+                                if (!verbTypes.includes(verb.verb)) {
+                                    continue;
+                                }
+                            }
+                        }
+                        // Verb passed filters - add to collection
+                        collectedVerbs.push(verb);
+                        // Early termination: stop when we have enough for offset + limit
+                        if (collectedVerbs.length >= targetCount) {
+                            break;
+                        }
+                    }
+                    totalScanned += verbsOfType.length;
+                }
+                catch (error) {
+                    // Ignore errors for types with no verbs (directory may not exist)
+                    // This is expected for types that haven't been used yet
+                }
+            }
+            // Apply pagination (slice for offset)
+            const paginatedVerbs = collectedVerbs.slice(offset, offset + limit);
+            const hasMore = collectedVerbs.length >= targetCount;
             return {
-                items: [],
-                totalCount: 0,
-                hasMore: false
+                items: paginatedVerbs,
+                totalCount: collectedVerbs.length, // Accurate count of filtered results
+                hasMore,
+                nextCursor: hasMore && paginatedVerbs.length > 0
+                    ? paginatedVerbs[paginatedVerbs.length - 1].id
+                    : undefined
             };
         }
         catch (error) {
@@ -1402,6 +1581,45 @@ export class BaseStorage extends BaseStorageAdapter {
         };
         await this.writeObjectToPath(`${SYSTEM_DIR}/type-statistics.json`, stats);
     }
+    /**
+     * Rebuild type counts from actual storage (v5.5.0)
+     * Called when statistics are missing or inconsistent
+     * Ensures verbCountsByType is always accurate for reliable pagination
+     */
+    async rebuildTypeCounts() {
+        console.log('[BaseStorage] Rebuilding type counts from storage...');
+        // Rebuild verb counts by checking each type directory
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const type = TypeUtils.getVerbFromIndex(i);
+            const prefix = `entities/verbs/${type}/vectors/`;
+            try {
+                const paths = await this.listObjectsInBranch(prefix);
+                this.verbCountsByType[i] = paths.length;
+            }
+            catch (error) {
+                // Type directory doesn't exist - count is 0
+                this.verbCountsByType[i] = 0;
+            }
+        }
+        // Rebuild noun counts similarly
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const type = TypeUtils.getNounFromIndex(i);
+            const prefix = `entities/nouns/${type}/vectors/`;
+            try {
+                const paths = await this.listObjectsInBranch(prefix);
+                this.nounCountsByType[i] = paths.length;
+            }
+            catch (error) {
+                // Type directory doesn't exist - count is 0
+                this.nounCountsByType[i] = 0;
+            }
+        }
+        // Save rebuilt counts to storage
+        await this.saveTypeStatistics();
+        const totalVerbs = this.verbCountsByType.reduce((sum, count) => sum + count, 0);
+        const totalNouns = this.nounCountsByType.reduce((sum, count) => sum + count, 0);
+        console.log(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs`);
+    }
     /**
      * Get noun type from cache or metadata
      * Relies on nounTypeCache populated during metadata saves

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "5.5.0",
+  "version": "5.6.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

package/dist/importManager.d.ts

@@ -1,78 +0,0 @@
-/**
- * Import Manager - Comprehensive data import with intelligent type detection
- *
- * Handles multiple data sources:
- * - Direct data (objects, arrays)
- * - Files (JSON, CSV, text)
- * - URLs (fetch and parse)
- * - Streams (for large files)
- *
- * Uses NeuralImportAugmentation for intelligent processing
- */
-import { NounType } from './types/graphTypes.js';
-export interface ImportOptions {
-    source?: 'data' | 'file' | 'url' | 'auto';
-    format?: 'json' | 'csv' | 'text' | 'yaml' | 'auto';
-    batchSize?: number;
-    autoDetect?: boolean;
-    typeHint?: NounType;
-    extractRelationships?: boolean;
-    csvDelimiter?: string;
-    csvHeaders?: boolean;
-    parallel?: boolean;
-    maxConcurrency?: number;
-}
-export interface ImportResult {
-    success: boolean;
-    nouns: string[];
-    verbs: string[];
-    errors: string[];
-    stats: {
-        total: number;
-        imported: number;
-        failed: number;
-        relationships: number;
-    };
-}
-export declare class ImportManager {
-    private neuralImport;
-    private typeMatcher;
-    private brain;
-    constructor(brain: any);
-    /**
-     * Initialize the import manager
-     */
-    init(): Promise<void>;
-    /**
-     * Main import method - handles all sources
-     */
-    import(source: string | Buffer | any[] | any, options?: ImportOptions): Promise<ImportResult>;
-    /**
-     * Import from file
-     */
-    importFile(filePath: string, options?: ImportOptions): Promise<ImportResult>;
-    /**
-     * Import from URL
-     */
-    importUrl(url: string, options?: ImportOptions): Promise<ImportResult>;
-    /**
-     * Detect source type
-     */
-    private detectSourceType;
-    /**
-     * Detect format from file path
-     */
-    private detectFormatFromPath;
-    /**
-     * Read file
-     */
-    private readFile;
-    /**
-     * Fetch from URL
-     */
-    private fetchFromUrl;
-}
-/**
- * Create an import manager instance
- */
-export declare function createImportManager(brain: any): ImportManager;

package/dist/importManager.js

@@ -1,267 +0,0 @@
-/**
- * Import Manager - Comprehensive data import with intelligent type detection
- *
- * Handles multiple data sources:
- * - Direct data (objects, arrays)
- * - Files (JSON, CSV, text)
- * - URLs (fetch and parse)
- * - Streams (for large files)
- *
- * Uses NeuralImportAugmentation for intelligent processing
- */
-import { VerbType } from './types/graphTypes.js';
-import { NeuralImportAugmentation } from './augmentations/neuralImport.js';
-import * as fs from './universal/fs.js';
-import * as path from './universal/path.js';
-import { prodLog } from './utils/logger.js';
-export class ImportManager {
-    constructor(brain) {
-        this.typeMatcher = null;
-        this.brain = brain;
-        this.neuralImport = new NeuralImportAugmentation();
-    }
-    /**
-     * Initialize the import manager
-     */
-    async init() {
-        // Initialize neural import with proper context
-        const context = {
-            brain: this.brain,
-            storage: this.brain.storage,
-            config: {},
-            log: (message, level) => {
-                if (level === 'error') {
-                    prodLog.error(message);
-                }
-                else if (level === 'warn') {
-                    prodLog.warn(message);
-                }
-                else {
-                    prodLog.info(message);
-                }
-            }
-        };
-        await this.neuralImport.initialize(context);
-        // Get type matcher
-        const { getBrainyTypes } = await import('./augmentations/typeMatching/brainyTypes.js');
-        this.typeMatcher = await getBrainyTypes();
-    }
-    /**
-     * Main import method - handles all sources
-     */
-    async import(source, options = {}) {
-        const result = {
-            success: false,
-            nouns: [],
-            verbs: [],
-            errors: [],
-            stats: {
-                total: 0,
-                imported: 0,
-                failed: 0,
-                relationships: 0
-            }
-        };
-        try {
-            // Detect source type
-            const sourceType = await this.detectSourceType(source, options.source);
-            // Get data based on source type
-            let data;
-            let format = options.format || 'auto';
-            switch (sourceType) {
-                case 'url':
-                    data = await this.fetchFromUrl(source);
-                    break;
-                case 'file':
-                    const filePath = source;
-                    data = await this.readFile(filePath);
-                    if (format === 'auto') {
-                        format = this.detectFormatFromPath(filePath);
-                    }
-                    break;
-                case 'data':
-                default:
-                    data = source;
-                    break;
-            }
-            // Process data through neural import
-            let items;
-            let relationships = [];
-            if (Buffer.isBuffer(data) || typeof data === 'string') {
-                // Use neural import for parsing and analysis
-                const analysis = await this.neuralImport.getNeuralAnalysis(data, format);
-                // Extract items and relationships
-                items = analysis.detectedEntities.map(entity => ({
-                    data: entity.originalData,
-                    type: entity.nounType,
-                    confidence: entity.confidence,
-                    id: entity.suggestedId
-                }));
-                if (options.extractRelationships !== false) {
-                    relationships = analysis.detectedRelationships;
-                }
-                // Log insights
-                for (const insight of analysis.insights) {
-                    prodLog.info(`🧠 ${insight.description} (confidence: ${insight.confidence})`);
-                }
-            }
-            else if (Array.isArray(data)) {
-                items = data;
-            }
-            else {
-                items = [data];
-            }
-            result.stats.total = items.length;
-            // Import items in batches
-            const batchSize = options.batchSize || 50;
-            for (let i = 0; i < items.length; i += batchSize) {
-                const batch = items.slice(i, i + batchSize);
-                // Process batch in parallel if enabled
-                const promises = batch.map(async (item) => {
-                    try {
-                        // Detect type if needed
-                        let nounType = item.type || options.typeHint;
-                        if (!nounType && options.autoDetect !== false && this.typeMatcher) {
-                            const match = await this.typeMatcher.matchNounType(item.data || item);
-                            nounType = match.type;
-                        }
-                        // Prepare the data to import
-                        const dataToImport = item.data || item;
-                        // Create metadata combining original data with import metadata
-                        const metadata = {
-                            ...(typeof dataToImport === 'object' ? dataToImport : {}),
-                            ...(item.data?.metadata || {}),
-                            nounType,
-                            _importedAt: new Date().toISOString(),
-                            _confidence: item.confidence
-                        };
-                        // Add to brain using modern API signature
-                        const id = await this.brain.add({ data: dataToImport, type: nounType || 'content', metadata });
-                        result.nouns.push(id);
-                        result.stats.imported++;
-                        return id;
-                    }
-                    catch (error) {
-                        result.errors.push(`Failed to import item: ${error.message}`);
-                        result.stats.failed++;
-                        return null;
-                    }
-                });
-                if (options.parallel !== false) {
-                    await Promise.all(promises);
-                }
-                else {
-                    for (const promise of promises) {
-                        await promise;
-                    }
-                }
-            }
-            // Import relationships
-            for (const rel of relationships) {
-                try {
-                    // Match verb type if needed
-                    let verbType = rel.verbType;
-                    if (!Object.values(VerbType).includes(verbType) && this.typeMatcher) {
-                        const match = await this.typeMatcher.matchVerbType({ id: rel.sourceId }, { id: rel.targetId }, rel.verbType);
-                        verbType = match.type;
-                    }
-                    const verbId = await this.brain.relate({
-                        from: rel.sourceId,
-                        to: rel.targetId,
-                        type: verbType,
-                        metadata: rel.metadata,
-                        weight: rel.weight
-                    });
-                    result.verbs.push(verbId);
-                    result.stats.relationships++;
-                }
-                catch (error) {
-                    result.errors.push(`Failed to create relationship: ${error.message}`);
-                }
-            }
-            result.success = result.stats.imported > 0;
-            prodLog.info(`✨ Import complete: ${result.stats.imported}/${result.stats.total} items, ${result.stats.relationships} relationships`);
-        }
-        catch (error) {
-            result.errors.push(`Import failed: ${error.message}`);
-            prodLog.error('Import failed:', error);
-        }
-        return result;
-    }
-    /**
-     * Import from file
-     */
-    async importFile(filePath, options = {}) {
-        return this.import(filePath, { ...options, source: 'file' });
-    }
-    /**
-     * Import from URL
-     */
-    async importUrl(url, options = {}) {
-        return this.import(url, { ...options, source: 'url' });
-    }
-    /**
-     * Detect source type
-     */
-    async detectSourceType(source, hint) {
-        if (hint && hint !== 'auto') {
-            return hint;
-        }
-        if (typeof source === 'string') {
-            // Check if URL
-            if (source.startsWith('http://') || source.startsWith('https://')) {
-                return 'url';
-            }
-            // Check if file path exists
-            try {
-                if (await fs.exists(source)) {
-                    return 'file';
-                }
-            }
-            catch (error) {
-                // File system check failed, not a file path
-                console.debug('File path check failed:', error);
-            }
-        }
-        return 'data';
-    }
-    /**
-     * Detect format from file path
-     */
-    detectFormatFromPath(filePath) {
-        const ext = path.extname(filePath).toLowerCase();
-        switch (ext) {
-            case '.json': return 'json';
-            case '.csv': return 'csv';
-            case '.txt': return 'text';
-            case '.md': return 'text';
-            case '.yaml':
-            case '.yml': return 'yaml';
-            default: return 'auto';
-        }
-    }
-    /**
-     * Read file
-     */
-    async readFile(filePath) {
-        const content = await fs.readFile(filePath, 'utf8');
-        return Buffer.from(content, 'utf8');
-    }
-    /**
-     * Fetch from URL
-     */
-    async fetchFromUrl(url) {
-        const response = await fetch(url);
-        if (!response.ok) {
-            throw new Error(`Failed to fetch ${url}: ${response.statusText}`);
-        }
-        return response.text();
-    }
-}
-/**
- * Create an import manager instance
- */
-export function createImportManager(brain) {
-    return new ImportManager(brain);
-}
-//# sourceMappingURL=importManager.js.map