@soulcraft/brainy 2.14.2 → 2.15.0

package/dist/brainyData.d.ts

@@ -1002,6 +1002,12 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
      * @returns Array of verbs of the specified type
      */
     getVerbsByType(type: string): Promise<GraphVerb[]>;
+    /**
+     * Get all verbs associated with a specific noun (both as source and target)
+     * @param nounId The ID of the noun
+     * @returns Array of verbs where the noun is either source or target
+     */
+    getVerbsForNoun(nounId: string): Promise<GraphVerb[]>;
     /**
      * Delete a verb
      * @param id The ID of the verb to delete

package/dist/brainyData.js

@@ -2985,6 +2985,30 @@ export class BrainyData {
             throw new Error(`Failed to get verbs by type ${type}: ${error}`);
         }
     }
+    /**
+     * Get all verbs associated with a specific noun (both as source and target)
+     * @param nounId The ID of the noun
+     * @returns Array of verbs where the noun is either source or target
+     */
+    async getVerbsForNoun(nounId) {
+        await this.ensureInitialized();
+        try {
+            // Get verbs where this noun is the source
+            const sourceVerbs = await this.getVerbsBySource(nounId);
+            // Get verbs where this noun is the target  
+            const targetVerbs = await this.getVerbsByTarget(nounId);
+            // Combine and deduplicate (in case a verb somehow appears in both - shouldn't happen but safety first)
+            const verbMap = new Map();
+            for (const verb of [...sourceVerbs, ...targetVerbs]) {
+                verbMap.set(verb.id, verb);
+            }
+            return Array.from(verbMap.values());
+        }
+        catch (error) {
+            console.error(`Failed to get verbs for noun ${nounId}:`, error);
+            throw error;
+        }
+    }
     /**
      * Delete a verb
      * @param id The ID of the verb to delete

package/dist/neural/improvedNeuralAPI.d.ts

@@ -18,7 +18,7 @@
  * Private methods are prefixed with _ and not exposed in public API
  */
 import { Vector } from '../coreTypes.js';
-import { SemanticCluster, DomainCluster, TemporalCluster, SimilarityOptions, SimilarityResult, NeighborOptions, NeighborsResult, SemanticHierarchy, HierarchyOptions, ClusteringOptions, DomainClusteringOptions, TemporalClusteringOptions, StreamClusteringOptions, VisualizationOptions, VisualizationResult, OutlierOptions, Outlier, StreamingBatch, TimeWindow, PerformanceMetrics, NeuralAPIConfig } from './types.js';
+import { SemanticCluster, DomainCluster, TemporalCluster, EnhancedSemanticCluster, SimilarityOptions, SimilarityResult, NeighborOptions, NeighborsResult, SemanticHierarchy, HierarchyOptions, ClusteringOptions, DomainClusteringOptions, TemporalClusteringOptions, StreamClusteringOptions, VisualizationOptions, VisualizationResult, OutlierOptions, Outlier, StreamingBatch, TimeWindow, PerformanceMetrics, NeuralAPIConfig } from './types.js';
 export declare class ImprovedNeuralAPI {
     private brain;
     private config;
@@ -71,6 +71,16 @@ export declare class ImprovedNeuralAPI {
      * Incremental clustering - add new items to existing clusters
      */
     updateClusters(newItems: string[], options?: ClusteringOptions): Promise<SemanticCluster[]>;
+    /**
+     * Enhanced clustering with relationship analysis using verbs
+     * Returns clusters with intra-cluster and inter-cluster relationship information
+     *
+     * Scalable for millions of nodes - uses efficient batching and filtering
+     */
+    clustersWithRelationships(input?: string | string[] | ClusteringOptions, options?: {
+        batchSize?: number;
+        maxRelationships?: number;
+    }): Promise<EnhancedSemanticCluster[]>;
     /**
      * Find K-nearest semantic neighbors
      */
@@ -276,6 +286,7 @@ export declare class ImprovedNeuralAPI {
     private _calculateItemToClusterSimilarity;
     private _recalculateClusterCentroid;
     private _calculateSimilarity;
+    private _calculateEdgeWeight;
     private _sortNeighbors;
     private _buildSemanticHierarchy;
     private _detectOutliersClusterBased;

package/dist/neural/improvedNeuralAPI.js

@@ -354,6 +354,161 @@ export class ImprovedNeuralAPI {
             throw new ClusteringError(`Failed to update clusters: ${errorMessage}`, { newItems, options });
         }
     }
+    /**
+     * Enhanced clustering with relationship analysis using verbs
+     * Returns clusters with intra-cluster and inter-cluster relationship information
+     *
+     * Scalable for millions of nodes - uses efficient batching and filtering
+     */
+    async clustersWithRelationships(input, options) {
+        const startTime = performance.now();
+        const batchSize = options?.batchSize || 1000;
+        const maxRelationships = options?.maxRelationships || 10000;
+        let processedCount = 0;
+        try {
+            // Get basic clusters first
+            const basicClusters = await this.clusters(input);
+            if (basicClusters.length === 0) {
+                return [];
+            }
+            // Build member lookup for O(1) cluster membership checking
+            const memberToClusterMap = new Map();
+            const clusterMap = new Map();
+            for (const cluster of basicClusters) {
+                clusterMap.set(cluster.id, cluster);
+                for (const memberId of cluster.members) {
+                    memberToClusterMap.set(memberId, cluster.id);
+                }
+            }
+            // Initialize cluster edge collections
+            const clusterEdges = new Map();
+            for (const cluster of basicClusters) {
+                clusterEdges.set(cluster.id, {
+                    intra: [],
+                    inter: [],
+                    edgeTypes: {}
+                });
+            }
+            // Process verbs in batches to handle millions of relationships efficiently
+            let hasMoreVerbs = true;
+            let offset = 0;
+            while (hasMoreVerbs && processedCount < maxRelationships) {
+                // Get batch of verbs using proper pagination API
+                const verbResult = await this.brain.getVerbs({
+                    pagination: {
+                        offset: offset,
+                        limit: batchSize
+                    }
+                });
+                const verbBatch = verbResult.data;
+                if (verbBatch.length === 0) {
+                    hasMoreVerbs = false;
+                    break;
+                }
+                // Process this batch
+                for (const verb of verbBatch) {
+                    if (processedCount >= maxRelationships)
+                        break;
+                    const sourceClusterId = memberToClusterMap.get(verb.sourceId);
+                    const targetClusterId = memberToClusterMap.get(verb.targetId);
+                    // Skip verbs that don't involve any clustered nodes
+                    if (!sourceClusterId && !targetClusterId)
+                        continue;
+                    const edgeWeight = this._calculateEdgeWeight(verb);
+                    const edgeType = verb.verb || verb.type || 'relationship';
+                    if (sourceClusterId && targetClusterId) {
+                        if (sourceClusterId === targetClusterId) {
+                            // Intra-cluster relationship
+                            const edges = clusterEdges.get(sourceClusterId);
+                            edges.intra.push({
+                                id: verb.id,
+                                source: verb.sourceId,
+                                target: verb.targetId,
+                                type: edgeType,
+                                weight: edgeWeight,
+                                isInterCluster: false,
+                                sourceCluster: sourceClusterId,
+                                targetCluster: sourceClusterId
+                            });
+                            edges.edgeTypes[edgeType] = (edges.edgeTypes[edgeType] || 0) + 1;
+                        }
+                        else {
+                            // Inter-cluster relationship
+                            const sourceEdges = clusterEdges.get(sourceClusterId);
+                            const targetEdges = clusterEdges.get(targetClusterId);
+                            const edge = {
+                                id: verb.id,
+                                source: verb.sourceId,
+                                target: verb.targetId,
+                                type: edgeType,
+                                weight: edgeWeight,
+                                isInterCluster: true,
+                                sourceCluster: sourceClusterId,
+                                targetCluster: targetClusterId
+                            };
+                            sourceEdges.inter.push(edge);
+                            // Don't duplicate - target cluster will see this as incoming
+                            sourceEdges.edgeTypes[edgeType] = (sourceEdges.edgeTypes[edgeType] || 0) + 1;
+                        }
+                    }
+                    else {
+                        // One-way relationship to/from cluster
+                        const clusterId = sourceClusterId || targetClusterId;
+                        const edges = clusterEdges.get(clusterId);
+                        edges.inter.push({
+                            id: verb.id,
+                            source: verb.sourceId,
+                            target: verb.targetId,
+                            type: edgeType,
+                            weight: edgeWeight,
+                            isInterCluster: true,
+                            sourceCluster: sourceClusterId || 'external',
+                            targetCluster: targetClusterId || 'external'
+                        });
+                        edges.edgeTypes[edgeType] = (edges.edgeTypes[edgeType] || 0) + 1;
+                    }
+                    processedCount++;
+                }
+                offset += batchSize;
+                // Memory management: if we have too many edges, break early
+                const totalEdges = Array.from(clusterEdges.values())
+                    .reduce((sum, edges) => sum + edges.intra.length + edges.inter.length, 0);
+                if (totalEdges >= maxRelationships) {
+                    console.warn(`Relationship analysis stopped at ${totalEdges} edges to maintain performance`);
+                    break;
+                }
+                // Check if we got fewer verbs than batch size (end of data)
+                if (verbBatch.length < batchSize) {
+                    hasMoreVerbs = false;
+                }
+            }
+            // Build enhanced clusters
+            const enhancedClusters = [];
+            for (const cluster of basicClusters) {
+                const edges = clusterEdges.get(cluster.id);
+                enhancedClusters.push({
+                    ...cluster,
+                    intraClusterEdges: edges.intra,
+                    interClusterEdges: edges.inter,
+                    relationshipSummary: {
+                        totalEdges: edges.intra.length + edges.inter.length,
+                        intraClusterEdges: edges.intra.length,
+                        interClusterEdges: edges.inter.length,
+                        edgeTypes: edges.edgeTypes
+                    }
+                });
+            }
+            this._trackPerformance('clustersWithRelationships', startTime, processedCount, 'enhanced-scalable');
+            return enhancedClusters;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new ClusteringError(`Failed to perform relationship-aware clustering: ${errorMessage}`, {
+                input: typeof input === 'object' ? JSON.stringify(input) : input,
+                processedCount: processedCount || 0
+            });
+        }
+    }
     // ===== PUBLIC API: NEIGHBORS & HIERARCHY =====
     /**
      * Find K-nearest semantic neighbors
@@ -2136,6 +2291,29 @@ The items were grouped using ${algorithm} clustering. What is the most appropria
     async _calculateSimilarity(id1, id2) {
         return await this.similar(id1, id2);
     }
+    _calculateEdgeWeight(verb) {
+        // Calculate edge weight based on verb properties
+        let weight = 1.0;
+        // Factor in connection strength if available
+        if (verb.connections && verb.connections instanceof Map) {
+            const connectionCount = verb.connections.size;
+            weight += Math.log(connectionCount + 1) * 0.1;
+        }
+        // Factor in verb type significance
+        const significantVerbs = ['caused', 'created', 'contains', 'implements', 'extends'];
+        if (verb.verb && significantVerbs.includes(verb.verb.toLowerCase())) {
+            weight += 0.3;
+        }
+        // Factor in recency if available
+        if (verb.metadata?.createdAt) {
+            const now = Date.now();
+            const created = new Date(verb.metadata.createdAt).getTime();
+            const daysSinceCreated = (now - created) / (1000 * 60 * 60 * 24);
+            // Newer relationships get slight boost
+            weight += Math.max(0, (30 - daysSinceCreated) / 100);
+        }
+        return Math.min(weight, 3.0); // Cap at 3.0
+    }
     _sortNeighbors(neighbors, sortBy) {
         switch (sortBy) {
             case 'similarity':

package/dist/neural/types.d.ts

@@ -17,6 +17,26 @@ export interface SemanticCluster {
     cohesion?: number;
     level?: number;
 }
+export interface ClusterEdge {
+    id: string;
+    source: string;
+    target: string;
+    type: string;
+    weight?: number;
+    isInterCluster: boolean;
+    sourceCluster?: string;
+    targetCluster?: string;
+}
+export interface EnhancedSemanticCluster extends SemanticCluster {
+    intraClusterEdges: ClusterEdge[];
+    interClusterEdges: ClusterEdge[];
+    relationshipSummary: {
+        totalEdges: number;
+        intraClusterEdges: number;
+        interClusterEdges: number;
+        edgeTypes: Record<string, number>;
+    };
+}
 export interface DomainCluster extends SemanticCluster {
     domain: string;
     domainConfidence: number;

package/dist/storage/adapters/fileSystemStorage.js

@@ -770,28 +770,40 @@ export class FileSystemStorage extends BaseStorage {
      * Get verbs by source
      */
     async getVerbsBySource_internal(sourceId) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getVerbsBySource_internal is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        console.log(`[DEBUG] getVerbsBySource_internal called for sourceId: ${sourceId}`);
+        // Use the working pagination method with source filter
+        const result = await this.getVerbsWithPagination({
+            limit: 10000,
+            filter: { sourceId: [sourceId] }
+        });
+        console.log(`[DEBUG] Found ${result.items.length} verbs for source ${sourceId}`);
+        return result.items;
     }
     /**
      * Get verbs by target
      */
     async getVerbsByTarget_internal(targetId) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getVerbsByTarget_internal is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        console.log(`[DEBUG] getVerbsByTarget_internal called for targetId: ${targetId}`);
+        // Use the working pagination method with target filter
+        const result = await this.getVerbsWithPagination({
+            limit: 10000,
+            filter: { targetId: [targetId] }
+        });
+        console.log(`[DEBUG] Found ${result.items.length} verbs for target ${targetId}`);
+        return result.items;
     }
     /**
      * Get verbs by type
      */
     async getVerbsByType_internal(type) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getVerbsByType_internal is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        console.log(`[DEBUG] getVerbsByType_internal called for type: ${type}`);
+        // Use the working pagination method with type filter
+        const result = await this.getVerbsWithPagination({
+            limit: 10000,
+            filter: { verbType: [type] }
+        });
+        console.log(`[DEBUG] Found ${result.items.length} verbs for type ${type}`);
+        return result.items;
     }
     /**
      * Get verbs with pagination
@@ -823,19 +835,57 @@ export class FileSystemStorage extends BaseStorage {
                 const file = verbFiles[i];
                 const id = file.replace('.json', '');
                 try {
-                    // Read the verb data
+                    // Read the verb data (HNSWVerb stored as edge)
                     const filePath = path.join(this.verbsDir, file);
                     const data = await fs.promises.readFile(filePath, 'utf-8');
                     const edge = JSON.parse(data);
-                    // Also try to get metadata if it exists
+                    // Get metadata which contains the actual verb information
                     const metadata = await this.getVerbMetadata(id);
-                    // Convert to GraphVerb format
+                    // If no metadata exists, try to reconstruct basic metadata from filename
+                    if (!metadata) {
+                        console.warn(`Verb ${id} has no metadata, trying to create minimal verb`);
+                        // Create minimal GraphVerb without full metadata
+                        const minimalVerb = {
+                            id: edge.id,
+                            vector: edge.vector,
+                            connections: edge.connections || new Map(),
+                            sourceId: 'unknown',
+                            targetId: 'unknown',
+                            source: 'unknown',
+                            target: 'unknown',
+                            type: 'relationship',
+                            verb: 'relatedTo'
+                        };
+                        verbs.push(minimalVerb);
+                        continue;
+                    }
+                    // Convert connections Map to proper format if needed
+                    let connections = edge.connections;
+                    if (connections && typeof connections === 'object' && !(connections instanceof Map)) {
+                        const connectionsMap = new Map();
+                        for (const [level, nodeIds] of Object.entries(connections)) {
+                            connectionsMap.set(Number(level), new Set(nodeIds));
+                        }
+                        connections = connectionsMap;
+                    }
+                    // Properly reconstruct GraphVerb from HNSWVerb + metadata
                     const verb = {
                         id: edge.id,
-                        source: metadata?.source || '',
-                        target: metadata?.target || '',
-                        type: metadata?.type || 'relationship',
-                        ...(metadata || {})
+                        vector: edge.vector, // Include the vector field!
+                        connections: connections,
+                        sourceId: metadata.sourceId || metadata.source,
+                        targetId: metadata.targetId || metadata.target,
+                        source: metadata.source || metadata.sourceId,
+                        target: metadata.target || metadata.targetId,
+                        verb: metadata.verb || metadata.type,
+                        type: metadata.type || metadata.verb,
+                        weight: metadata.weight,
+                        metadata: metadata.metadata || metadata,
+                        data: metadata.data,
+                        createdAt: metadata.createdAt,
+                        updatedAt: metadata.updatedAt,
+                        createdBy: metadata.createdBy,
+                        embedding: metadata.embedding || edge.vector
                     };
                     // Apply filters if provided
                     if (options.filter) {
@@ -843,19 +893,22 @@ export class FileSystemStorage extends BaseStorage {
                         // Check verbType filter
                         if (filter.verbType) {
                             const types = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
-                            if (!types.includes(verb.type || ''))
+                            const verbType = verb.type || verb.verb;
+                            if (verbType && !types.includes(verbType))
                                 continue;
                         }
                         // Check sourceId filter
                         if (filter.sourceId) {
                             const sources = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId];
-                            if (!sources.includes(verb.source || ''))
+                            const sourceId = verb.sourceId || verb.source;
+                            if (!sourceId || !sources.includes(sourceId))
                                 continue;
                         }
                         // Check targetId filter
                         if (filter.targetId) {
                             const targets = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId];
-                            if (!targets.includes(verb.target || ''))
+                            const targetId = verb.targetId || verb.target;
+                            if (!targetId || !targets.includes(targetId))
                                 continue;
                         }
                         // Check service filter

package/dist/storage/baseStorage.js

@@ -133,9 +133,31 @@ export class BaseStorage extends BaseStorageAdapter {
             createdBy: verb.createdBy,
             embedding: verb.embedding
         };
-        // Save both the HNSWVerb and metadata
-        await this.saveVerb_internal(hnswVerb);
-        await this.saveVerbMetadata(verb.id, metadata);
+        // Save both the HNSWVerb and metadata atomically
+        try {
+            console.log(`[DEBUG] Saving verb ${verb.id}: sourceId=${verb.sourceId}, targetId=${verb.targetId}`);
+            // Save the HNSWVerb first
+            await this.saveVerb_internal(hnswVerb);
+            console.log(`[DEBUG] Successfully saved HNSWVerb file for ${verb.id}`);
+            // Then save the metadata
+            await this.saveVerbMetadata(verb.id, metadata);
+            console.log(`[DEBUG] Successfully saved metadata file for ${verb.id}`);
+        }
+        catch (error) {
+            console.error(`[ERROR] Failed to save verb ${verb.id}:`, error);
+            // Attempt cleanup - remove verb file if metadata failed
+            try {
+                const verbExists = await this.getVerb_internal(verb.id);
+                if (verbExists) {
+                    console.log(`[CLEANUP] Attempting to remove orphaned verb file ${verb.id}`);
+                    await this.deleteVerb_internal(verb.id);
+                }
+            }
+            catch (cleanupError) {
+                console.error(`[ERROR] Failed to cleanup orphaned verb ${verb.id}:`, cleanupError);
+            }
+            throw new Error(`Failed to save verb ${verb.id}: ${error instanceof Error ? error.message : String(error)}`);
+        }
     }
     /**
      * Get a verb from storage

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "2.14.2",
+  "version": "2.15.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",