@soulcraft/brainy 2.11.0 → 2.14.0

package/dist/neural/improvedNeuralAPI.js

@@ -0,0 +1,2439 @@
+/**
+ * Improved Neural API - Clean, Consistent, Performant
+ *
+ * Public API Surface:
+ * - brain.neural.similar(a, b, options?)           // Similarity calculation
+ * - brain.neural.clusters(items?, options?)        // Semantic clustering
+ * - brain.neural.neighbors(id, options?)           // K-nearest neighbors
+ * - brain.neural.hierarchy(id, options?)           // Semantic hierarchy
+ * - brain.neural.outliers(options?)                // Anomaly detection
+ * - brain.neural.visualize(options?)               // Visualization data
+ *
+ * Advanced Clustering:
+ * - brain.neural.clusterByDomain(field, options?)  // Domain-aware clustering
+ * - brain.neural.clusterByTime(field, windows, options?) // Temporal clustering
+ * - brain.neural.clusterStream(options?)           // AsyncIterator for streaming
+ * - brain.neural.updateClusters(items, options?)   // Incremental clustering
+ *
+ * Private methods are prefixed with _ and not exposed in public API
+ */
+import { cosineDistance, euclideanDistance } from '../utils/distance.js';
+import { NeuralAPIError, ClusteringError, SimilarityError } from './types.js';
+export class ImprovedNeuralAPI {
+    constructor(brain, config = {}) {
+        // Caching for performance
+        this.similarityCache = new Map();
+        this.clusterCache = new Map();
+        this.hierarchyCache = new Map();
+        this.neighborsCache = new Map();
+        // Performance tracking
+        this.performanceMetrics = new Map();
+        this.brain = brain;
+        this.config = {
+            cacheSize: 1000,
+            defaultAlgorithm: 'auto',
+            similarityMetric: 'cosine',
+            performanceTracking: true,
+            maxMemoryUsage: '1GB',
+            parallelProcessing: true,
+            streamingBatchSize: 100,
+            ...config
+        };
+        this._initializeCleanupTimer();
+    }
+    // ===== PUBLIC API: SIMILARITY =====
+    /**
+     * Calculate similarity between any two items (auto-detection)
+     * Supports: IDs, text strings, vectors, or mixed types
+     */
+    async similar(a, b, options = {}) {
+        const startTime = performance.now();
+        try {
+            // Create cache key
+            const cacheKey = this._createSimilarityKey(a, b, options);
+            if (this.similarityCache.has(cacheKey)) {
+                return this.similarityCache.get(cacheKey);
+            }
+            let result;
+            // Auto-detect input types and route accordingly
+            if (this._isId(a) && this._isId(b)) {
+                result = await this._similarityById(a, b, options);
+            }
+            else if (this._isVector(a) && this._isVector(b)) {
+                result = await this._similarityByVector(a, b, options);
+            }
+            else if (typeof a === 'string' && typeof b === 'string') {
+                result = await this._similarityByText(a, b, options);
+            }
+            else {
+                // Mixed types - convert to vectors
+                const vectorA = await this._convertToVector(a);
+                const vectorB = await this._convertToVector(b);
+                result = await this._similarityByVector(vectorA, vectorB, options);
+            }
+            // Cache result
+            this._cacheResult(cacheKey, result, this.similarityCache);
+            // Track performance
+            this._trackPerformance('similarity', startTime, 2, 'mixed');
+            return result;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new SimilarityError(`Failed to calculate similarity: ${errorMessage}`, {
+                inputA: typeof a === 'object' ? 'vector' : String(a).substring(0, 50),
+                inputB: typeof b === 'object' ? 'vector' : String(b).substring(0, 50),
+                options
+            });
+        }
+    }
+    // ===== PUBLIC API: CLUSTERING =====
+    /**
+     * Intelligent semantic clustering with auto-routing
+     * - No input: Cluster all data
+     * - Array: Cluster specific items
+     * - String: Find clusters near this item
+     * - Options object: Advanced configuration
+     */
+    async clusters(input) {
+        const startTime = performance.now();
+        try {
+            let options = {};
+            let items;
+            // Parse input
+            if (!input) {
+                // Cluster all data
+                items = undefined;
+                options = { algorithm: 'auto' };
+            }
+            else if (Array.isArray(input)) {
+                // Cluster specific items
+                items = input;
+                options = { algorithm: 'auto' };
+            }
+            else if (typeof input === 'string') {
+                // Find clusters near this item
+                const nearbyResult = await this.neighbors(input, { limit: 100 });
+                items = nearbyResult.neighbors.map(n => n.id);
+                options = { algorithm: 'auto' };
+            }
+            else if (typeof input === 'object') {
+                // Configuration object
+                options = input;
+                items = undefined;
+            }
+            else {
+                throw new ClusteringError('Invalid input for clustering', { input });
+            }
+            // Check cache
+            const cacheKey = this._createClusteringKey(items, options);
+            if (this.clusterCache.has(cacheKey)) {
+                const cached = this.clusterCache.get(cacheKey);
+                return cached.clusters;
+            }
+            // Route to optimal algorithm
+            const result = await this._routeClusteringAlgorithm(items, options);
+            // Cache result
+            this._cacheResult(cacheKey, result, this.clusterCache);
+            // Track performance
+            this._trackPerformance('clustering', startTime, items?.length || 0, options.algorithm || 'auto');
+            return result.clusters;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new ClusteringError(`Failed to perform clustering: ${errorMessage}`, {
+                input: typeof input === 'object' ? JSON.stringify(input) : input,
+            });
+        }
+    }
+    /**
+     * Fast hierarchical clustering using HNSW levels
+     */
+    async clusterFast(options = {}) {
+        const fullOptions = {
+            algorithm: 'hierarchical',
+            maxClusters: options.maxClusters,
+            ...options
+        };
+        const result = await this._performHierarchicalClustering(undefined, fullOptions);
+        return result.clusters;
+    }
+    /**
+     * Large-scale clustering with intelligent sampling
+     */
+    async clusterLarge(options = {}) {
+        const fullOptions = {
+            algorithm: 'auto',
+            sampleSize: options.sampleSize || 1000,
+            strategy: options.strategy || 'diverse',
+            ...options
+        };
+        const result = await this._performSampledClustering(undefined, fullOptions);
+        return result.clusters;
+    }
+    // ===== PUBLIC API: ADVANCED CLUSTERING =====
+    /**
+     * Domain-aware clustering based on metadata fields
+     */
+    async clusterByDomain(field, options = {}) {
+        const startTime = performance.now();
+        try {
+            // Get all items with the specified field
+            const items = await this._getItemsByField(field);
+            if (items.length === 0) {
+                return [];
+            }
+            // Group by domain values
+            const domainGroups = this._groupByDomain(items, field);
+            const domainClusters = [];
+            // Cluster within each domain
+            for (const [domain, domainItems] of domainGroups) {
+                const domainOptions = {
+                    ...options,
+                    algorithm: 'auto',
+                    maxClusters: Math.min(options.maxClusters || 10, Math.ceil(domainItems.length / 3))
+                };
+                const clusters = await this._performClustering(domainItems.map(item => item.id), domainOptions);
+                // Convert to domain clusters
+                for (const cluster of clusters.clusters) {
+                    domainClusters.push({
+                        ...cluster,
+                        domain,
+                        domainConfidence: this._calculateDomainConfidence(cluster, domainItems),
+                        crossDomainMembers: options.crossDomainThreshold
+                            ? await this._findCrossDomainMembers(cluster, options.crossDomainThreshold)
+                            : undefined
+                    });
+                }
+            }
+            // Handle cross-domain clustering if enabled
+            if (!options.preserveDomainBoundaries) {
+                const crossDomainClusters = await this._findCrossDomainClusters(domainClusters, options.crossDomainThreshold || 0.8);
+                domainClusters.push(...crossDomainClusters);
+            }
+            this._trackPerformance('domainClustering', startTime, items.length, field);
+            return domainClusters;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new ClusteringError(`Failed to cluster by domain: ${errorMessage}`, { field, options });
+        }
+    }
+    /**
+     * Temporal clustering based on time windows
+     */
+    async clusterByTime(timeField, windows, options = { timeField, windows }) {
+        const startTime = performance.now();
+        try {
+            const temporalClusters = [];
+            for (const window of windows) {
+                // Get items in this time window
+                const windowItems = await this._getItemsByTimeWindow(timeField, window);
+                if (windowItems.length === 0)
+                    continue;
+                // Cluster items in this window  
+                const clusteringOptions = {
+                    ...options,
+                    algorithm: 'auto'
+                };
+                const clusters = await this._performClustering(windowItems.map(item => item.id), clusteringOptions);
+                // Convert to temporal clusters
+                for (const cluster of clusters.clusters) {
+                    const temporal = await this._calculateTemporalMetrics(cluster, windowItems, timeField);
+                    temporalClusters.push({
+                        ...cluster,
+                        timeWindow: window,
+                        trend: temporal.trend,
+                        temporal: temporal.metrics
+                    });
+                }
+            }
+            // Handle overlapping windows
+            if (options.overlapStrategy === 'merge') {
+                return this._mergeOverlappingTemporalClusters(temporalClusters);
+            }
+            this._trackPerformance('temporalClustering', startTime, temporalClusters.length, 'temporal');
+            return temporalClusters;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new ClusteringError(`Failed to cluster by time: ${errorMessage}`, { timeField, windows, options });
+        }
+    }
+    /**
+     * Streaming clustering with real-time updates
+     */
+    async *clusterStream(options = {}) {
+        const batchSize = options.batchSize || this.config.streamingBatchSize || 100;
+        let batchNumber = 0;
+        let processedCount = 0;
+        try {
+            // Get all items for processing
+            const allItems = await this._getAllItemIds();
+            const totalItems = allItems.length;
+            // Process in batches
+            for (let i = 0; i < allItems.length; i += batchSize) {
+                const startTime = performance.now();
+                const batch = allItems.slice(i, i + batchSize);
+                // Perform clustering on this batch
+                const result = await this._performClustering(batch, {
+                    ...options,
+                    algorithm: 'auto',
+                    cacheResults: false // Don't cache streaming results
+                });
+                processedCount += batch.length;
+                const isComplete = processedCount >= totalItems;
+                yield {
+                    clusters: result.clusters,
+                    batchNumber: ++batchNumber,
+                    isComplete,
+                    progress: {
+                        processed: processedCount,
+                        total: totalItems,
+                        percentage: (processedCount / totalItems) * 100
+                    },
+                    metrics: {
+                        ...result.metrics,
+                        executionTime: performance.now() - startTime
+                    }
+                };
+                // Adaptive threshold adjustment
+                if (options.adaptiveThreshold && batchNumber > 1) {
+                    options.threshold = this._adjustThresholdAdaptively(result.clusters, options.threshold);
+                }
+            }
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new ClusteringError(`Failed in streaming clustering: ${errorMessage}`, { options, batchNumber });
+        }
+    }
+    /**
+     * Incremental clustering - add new items to existing clusters
+     */
+    async updateClusters(newItems, options = {}) {
+        const startTime = performance.now();
+        try {
+            // Get existing clusters
+            const existingClusters = await this.clusters({ ...options, algorithm: 'auto' });
+            // For each new item, find best cluster or create new one
+            const updatedClusters = [...existingClusters];
+            const unassignedItems = [];
+            for (const itemId of newItems) {
+                let bestCluster = null;
+                let bestSimilarity = 0;
+                // Find most similar existing cluster
+                for (const cluster of updatedClusters) {
+                    const similarity = await this._calculateItemToClusterSimilarity(itemId, cluster);
+                    if (similarity > bestSimilarity && similarity > (options.threshold || 0.6)) {
+                        bestSimilarity = similarity;
+                        bestCluster = cluster;
+                    }
+                }
+                if (bestCluster) {
+                    // Add to existing cluster
+                    bestCluster.members.push(itemId);
+                    bestCluster.size = bestCluster.members.length;
+                    // Recalculate centroid
+                    bestCluster.centroid = await this._recalculateClusterCentroid(bestCluster);
+                }
+                else {
+                    // Item doesn't fit existing clusters
+                    unassignedItems.push(itemId);
+                }
+            }
+            // Create new clusters for unassigned items
+            if (unassignedItems.length > 0) {
+                const newClusters = await this._performClustering(unassignedItems, options);
+                updatedClusters.push(...newClusters.clusters);
+            }
+            this._trackPerformance('incrementalClustering', startTime, newItems.length, 'incremental');
+            return updatedClusters;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new ClusteringError(`Failed to update clusters: ${errorMessage}`, { newItems, options });
+        }
+    }
+    // ===== PUBLIC API: NEIGHBORS & HIERARCHY =====
+    /**
+     * Find K-nearest semantic neighbors
+     */
+    async neighbors(id, options = {}) {
+        const startTime = performance.now();
+        try {
+            const cacheKey = `neighbors:${id}:${JSON.stringify(options)}`;
+            if (this.neighborsCache.has(cacheKey)) {
+                return this.neighborsCache.get(cacheKey);
+            }
+            const limit = options.limit || 10;
+            const minSimilarity = options.minSimilarity || 0.1;
+            // Use HNSW index for efficient neighbor search
+            const searchResults = await this.brain.search('', {
+                ...options,
+                limit: limit * 2, // Get more than needed for filtering
+                metadata: options.includeMetadata ? {} : undefined
+            });
+            // Filter and sort neighbors
+            const neighbors = [];
+            for (const result of searchResults) {
+                if (result.id === id)
+                    continue; // Skip self
+                const similarity = await this._calculateSimilarity(id, result.id);
+                if (similarity >= minSimilarity) {
+                    neighbors.push({
+                        id: result.id,
+                        similarity,
+                        data: result.content || result.data,
+                        metadata: options.includeMetadata ? result.metadata : undefined,
+                        distance: 1 - similarity
+                    });
+                }
+                if (neighbors.length >= limit)
+                    break;
+            }
+            // Sort by specified criteria
+            this._sortNeighbors(neighbors, options.sortBy || 'similarity');
+            const result = {
+                neighbors: neighbors.slice(0, limit),
+                queryId: id,
+                totalFound: neighbors.length,
+                averageSimilarity: neighbors.reduce((sum, n) => sum + n.similarity, 0) / neighbors.length
+            };
+            this._cacheResult(cacheKey, result, this.neighborsCache);
+            this._trackPerformance('neighbors', startTime, limit, 'knn');
+            return result;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new NeuralAPIError(`Failed to find neighbors: ${errorMessage}`, 'NEIGHBORS_ERROR', { id, options });
+        }
+    }
+    /**
+     * Build semantic hierarchy around an item
+     */
+    async hierarchy(id, options = {}) {
+        const startTime = performance.now();
+        try {
+            const cacheKey = `hierarchy:${id}:${JSON.stringify(options)}`;
+            if (this.hierarchyCache.has(cacheKey)) {
+                return this.hierarchyCache.get(cacheKey);
+            }
+            // Get item data
+            const item = await this.brain.getNoun(id);
+            if (!item) {
+                throw new Error(`Item with ID ${id} not found`);
+            }
+            // Build hierarchy based on strategy
+            const hierarchy = await this._buildSemanticHierarchy(item, options);
+            this._cacheResult(cacheKey, hierarchy, this.hierarchyCache);
+            this._trackPerformance('hierarchy', startTime, 1, 'hierarchy');
+            return hierarchy;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new NeuralAPIError(`Failed to build hierarchy: ${errorMessage}`, 'HIERARCHY_ERROR', { id, options });
+        }
+    }
+    // ===== PUBLIC API: ANALYSIS =====
+    /**
+     * Detect outliers and anomalous items
+     */
+    async outliers(options = {}) {
+        const startTime = performance.now();
+        try {
+            const threshold = options.threshold || 0.3;
+            const method = options.method || 'cluster-based';
+            let outliers = [];
+            switch (method) {
+                case 'isolation':
+                    outliers = await this._detectOutliersIsolation(threshold, options);
+                    break;
+                case 'statistical':
+                    outliers = await this._detectOutliersStatistical(threshold, options);
+                    break;
+                case 'cluster-based':
+                default:
+                    outliers = await this._detectOutliersClusterBased(threshold, options);
+                    break;
+            }
+            this._trackPerformance('outlierDetection', startTime, outliers.length, method);
+            return outliers;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new NeuralAPIError(`Failed to detect outliers: ${errorMessage}`, 'OUTLIER_ERROR', { options });
+        }
+    }
+    /**
+     * Generate visualization data for graph libraries
+     */
+    async visualize(options = {}) {
+        const startTime = performance.now();
+        try {
+            const maxNodes = options.maxNodes || 100;
+            const dimensions = options.dimensions || 2;
+            const algorithm = options.algorithm || 'force';
+            // Get data for visualization
+            const nodes = await this._generateVisualizationNodes(maxNodes, options);
+            const edges = options.includeEdges ? await this._generateVisualizationEdges(nodes, options) : [];
+            const clusters = options.clusterColors ? await this._generateVisualizationClusters(nodes) : [];
+            // Apply layout algorithm
+            const positionedNodes = await this._applyLayoutAlgorithm(nodes, edges, algorithm, dimensions);
+            const result = {
+                nodes: positionedNodes,
+                edges,
+                clusters,
+                metadata: {
+                    algorithm,
+                    dimensions,
+                    totalNodes: nodes.length,
+                    totalEdges: edges.length,
+                    generatedAt: new Date()
+                }
+            };
+            this._trackPerformance('visualization', startTime, nodes.length, algorithm);
+            return result;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new NeuralAPIError(`Failed to generate visualization: ${errorMessage}`, 'VISUALIZATION_ERROR', { options });
+        }
+    }
+    // ===== PRIVATE IMPLEMENTATION METHODS =====
+    async _routeClusteringAlgorithm(items, options) {
+        const algorithm = options.algorithm || 'auto';
+        const itemCount = items?.length || await this._getTotalItemCount();
+        // Auto-select optimal algorithm based on data size and characteristics
+        if (algorithm === 'auto') {
+            // Intelligent algorithm selection based on data characteristics
+            const itemIds = items || await this._getAllItemIds();
+            const dataCharacteristics = await this._analyzeDataCharacteristics(itemIds);
+            const hasRichGraph = dataCharacteristics.graphDensity > 0.05;
+            const hasSemanticTypes = Object.keys(dataCharacteristics.typeDistribution).length > 3;
+            if (hasRichGraph && hasSemanticTypes) {
+                // Best of all worlds for rich semantic graphs
+                return this._performMultiModalClustering(items, { ...options, algorithm: 'multimodal' });
+            }
+            else if (hasRichGraph) {
+                // Strong relationship network - use graph clustering
+                return this._performGraphClustering(items, { ...options, algorithm: 'graph' });
+            }
+            else if (hasSemanticTypes) {
+                // Rich semantic taxonomy - use semantic clustering
+                return this._performSemanticClustering(items, { ...options, algorithm: 'semantic' });
+            }
+            else if (itemCount > 10000) {
+                // Large dataset - use sampling
+                return this._performSampledClustering(items, { ...options, algorithm: 'sample' });
+            }
+            else if (itemCount > 1000) {
+                // Medium dataset - use hierarchical HNSW
+                return this._performHierarchicalClustering(items, { ...options, algorithm: 'hierarchical' });
+            }
+            else {
+                // Small dataset - use k-means for quality
+                return this._performKMeansClustering(items, { ...options, algorithm: 'kmeans' });
+            }
+        }
+        // Use specified algorithm
+        switch (algorithm) {
+            case 'hierarchical':
+                return this._performHierarchicalClustering(items, options);
+            case 'semantic':
+                return this._performSemanticClustering(items, options);
+            case 'graph':
+                return this._performGraphClustering(items, options);
+            case 'multimodal':
+                return this._performMultiModalClustering(items, options);
+            case 'kmeans':
+                return this._performKMeansClustering(items, options);
+            case 'dbscan':
+                return this._performDBSCANClustering(items, options);
+            case 'sample':
+                return this._performSampledClustering(items, options);
+            default:
+                throw new ClusteringError(`Unsupported algorithm: ${algorithm}`);
+        }
+    }
+    async _performClustering(items, options) {
+        // This is the main clustering dispatcher - routes to specific algorithms
+        return this._routeClusteringAlgorithm(items, options);
+    }
+    // ===== REAL CLUSTERING IMPLEMENTATIONS =====
+    /**
+     * SEMANTIC-AWARE CLUSTERING: Uses existing NounType/VerbType taxonomy + HNSW
+     */
+    async _performSemanticClustering(items, options) {
+        const startTime = performance.now();
+        // Get all items if not specified
+        const itemIds = items || await this._getAllItemIds();
+        if (itemIds.length === 0) {
+            return this._createEmptyResult(startTime, 'semantic');
+        }
+        // 1. Group items by semantic type (NounType) - O(n) operation
+        const itemsWithMetadata = await this._getItemsWithMetadata(itemIds);
+        const typeGroups = this._groupBySemanticType(itemsWithMetadata);
+        const allClusters = [];
+        // 2. Cluster within each semantic type using HNSW - parallel processing
+        const typeClusteringPromises = Array.from(typeGroups.entries()).map(async ([nounType, groupItems]) => {
+            if (groupItems.length < (options.minClusterSize || 2)) {
+                // Create single cluster for small groups
+                return [{
+                        id: `semantic-${nounType}`,
+                        centroid: await this._calculateGroupCentroid(groupItems),
+                        members: groupItems.map(item => item.id),
+                        size: groupItems.length,
+                        confidence: 0.9, // High confidence for type-based clustering
+                        label: `${nounType} cluster`,
+                        metadata: { semanticType: nounType, clustering: 'semantic' }
+                    }];
+            }
+            // Use HNSW hierarchical clustering within type
+            return this._clusterWithinSemanticType(groupItems, options);
+        });
+        const typeClusterResults = await Promise.all(typeClusteringPromises);
+        typeClusterResults.forEach(clusters => allClusters.push(...clusters));
+        // 3. Find cross-type relationships using existing verb connections
+        const crossTypeConnections = await this._findCrossTypeConnections(typeGroups, options);
+        // 4. Merge clusters that have strong cross-type relationships
+        const finalClusters = await this._mergeSemanticClusters(allClusters, crossTypeConnections);
+        return {
+            clusters: finalClusters.slice(0, options.maxClusters || finalClusters.length),
+            metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'semantic'),
+            metadata: {
+                totalItems: itemIds.length,
+                clustersFound: finalClusters.length,
+                averageClusterSize: finalClusters.reduce((sum, c) => sum + c.size, 0) / finalClusters.length || 0,
+                semanticTypes: Array.from(typeGroups.keys()).length,
+                timestamp: new Date()
+            }
+        };
+    }
+    /**
+     * HIERARCHICAL CLUSTERING: Uses existing HNSW levels for O(n) clustering
+     */
+    async _performHierarchicalClustering(items, options) {
+        const startTime = performance.now();
+        const itemIds = items || await this._getAllItemIds();
+        if (itemIds.length === 0) {
+            return this._createEmptyResult(startTime, 'hierarchical');
+        }
+        // Use existing HNSW level structure for natural clustering
+        const level = options.level || this._getOptimalClusteringLevel(itemIds.length);
+        const maxClusters = options.maxClusters || Math.min(50, Math.ceil(itemIds.length / 20));
+        // Get HNSW level representatives - these are natural cluster centers
+        const levelNodes = await this._getHNSWLevelNodes(level);
+        const clusterCenters = levelNodes.slice(0, maxClusters);
+        const clusters = [];
+        // Create clusters around each level representative
+        for (let i = 0; i < clusterCenters.length; i++) {
+            const center = clusterCenters[i];
+            // Find items that belong to this cluster using HNSW neighbors
+            const members = await this._findClusterMembers(center, itemIds, 0.5);
+            if (members.length > 0) {
+                // Get actual node data for creating cluster
+                const memberData = await this._getItemsWithMetadata(members);
+                const centroid = await this._calculateCentroidFromItems(memberData);
+                clusters.push({
+                    id: `hierarchical-${i}`,
+                    centroid,
+                    members,
+                    size: members.length,
+                    confidence: await this._calculateHierarchicalConfidence(members),
+                    label: await this._generateClusterLabel(memberData, 'hierarchical'),
+                    metadata: { level, clusterCenter: center, clustering: 'hierarchical' }
+                });
+            }
+        }
+        // Assign remaining items to nearest clusters
+        const assignedItems = new Set(clusters.flatMap(c => c.members));
+        const unassignedItems = itemIds.filter(id => !assignedItems.has(id));
+        if (unassignedItems.length > 0) {
+            await this._assignUnassignedItems(unassignedItems, clusters);
+        }
+        return {
+            clusters,
+            metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'hierarchical'),
+            metadata: {
+                totalItems: itemIds.length,
+                clustersFound: clusters.length,
+                averageClusterSize: clusters.reduce((sum, c) => sum + c.size, 0) / clusters.length || 0,
+                hnswLevel: level,
+                timestamp: new Date()
+            }
+        };
+    }
+    /**
+     * K-MEANS CLUSTERING: Real implementation using existing distance functions
+     */
+    async _performKMeansClustering(items, options) {
+        const startTime = performance.now();
+        const itemIds = items || await this._getAllItemIds();
+        if (itemIds.length === 0) {
+            return this._createEmptyResult(startTime, 'kmeans');
+        }
+        // Get vectors for all items using existing infrastructure
+        const itemsWithVectors = await this._getItemsWithVectors(itemIds);
+        // Determine optimal k
+        const k = options.maxClusters || Math.min(Math.floor(Math.sqrt(itemsWithVectors.length / 2)), 50 // Maximum clusters for practical use
+        );
+        if (k <= 1) {
+            // Single cluster case
+            return {
+                clusters: [{
+                        id: 'kmeans-single',
+                        centroid: await this._calculateCentroidFromItems(itemsWithVectors),
+                        members: itemIds,
+                        size: itemIds.length,
+                        confidence: 1.0,
+                        label: 'Single cluster',
+                        metadata: { clustering: 'kmeans', k: 1 }
+                    }],
+                metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'kmeans'),
+                metadata: {
+                    totalItems: itemIds.length,
+                    clustersFound: 1,
+                    averageClusterSize: itemIds.length,
+                    kValue: 1,
+                    timestamp: new Date()
+                }
+            };
+        }
+        // Initialize centroids using k-means++ for better convergence
+        const centroids = await this._initializeCentroidsKMeansPlusPlus(itemsWithVectors, k);
+        let assignments = new Array(itemsWithVectors.length).fill(0);
+        let hasConverged = false;
+        const maxIterations = options.maxIterations || 100;
+        const tolerance = options.tolerance || 1e-4;
+        // K-means iteration loop
+        for (let iteration = 0; iteration < maxIterations && !hasConverged; iteration++) {
+            // Assignment step: assign each point to nearest centroid
+            const newAssignments = await this._assignPointsToCentroids(itemsWithVectors, centroids);
+            // Update step: recalculate centroids
+            const newCentroids = await this._updateCentroids(itemsWithVectors, newAssignments, k);
+            // Check convergence: has assignment changed significantly?
+            const changeRate = this._calculateAssignmentChangeRate(assignments, newAssignments);
+            hasConverged = changeRate < tolerance;
+            assignments = newAssignments;
+            // Update centroids for next iteration
+            for (let i = 0; i < centroids.length; i++) {
+                centroids[i] = newCentroids[i];
+            }
+        }
+        // Create semantic clusters from k-means results
+        const clusters = [];
+        for (let clusterIndex = 0; clusterIndex < k; clusterIndex++) {
+            const clusterMembers = itemsWithVectors.filter((_, i) => assignments[i] === clusterIndex);
+            if (clusterMembers.length > 0) {
+                const memberIds = clusterMembers.map(item => item.id);
+                clusters.push({
+                    id: `kmeans-${clusterIndex}`,
+                    centroid: centroids[clusterIndex],
+                    members: memberIds,
+                    size: memberIds.length,
+                    confidence: await this._calculateKMeansClusterConfidence(clusterMembers, centroids[clusterIndex]),
+                    label: await this._generateClusterLabel(clusterMembers, 'kmeans'),
+                    metadata: {
+                        clustering: 'kmeans',
+                        k,
+                        clusterIndex,
+                        convergenceIterations: maxIterations
+                    }
+                });
+            }
+        }
+        return {
+            clusters,
+            metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'kmeans'),
+            metadata: {
+                totalItems: itemIds.length,
+                clustersFound: clusters.length,
+                averageClusterSize: clusters.reduce((sum, c) => sum + c.size, 0) / clusters.length || 0,
+                kValue: k,
+                hasConverged,
+                timestamp: new Date()
+            }
+        };
+    }
+    /**
+     * DBSCAN CLUSTERING: Density-based clustering with adaptive parameters using HNSW
+     */
+    async _performDBSCANClustering(items, options) {
+        const startTime = performance.now();
+        const itemIds = items || await this._getAllItemIds();
+        if (itemIds.length === 0) {
+            return this._createEmptyResult(startTime, 'dbscan');
+        }
+        const itemsWithVectors = await this._getItemsWithVectors(itemIds);
+        // Adaptive parameter selection using HNSW neighbors
+        const minPts = options.minClusterSize || Math.max(4, Math.floor(Math.log2(itemsWithVectors.length)));
+        const eps = options.threshold || await this._estimateOptimalEps(itemsWithVectors, minPts);
+        // DBSCAN state tracking
+        const NOISE = -1;
+        const UNVISITED = 0;
+        const visited = new Map();
+        const clusterAssignments = new Map();
+        let currentClusterId = 1;
+        // Process each point
+        for (const item of itemsWithVectors) {
+            if (visited.get(item.id))
+                continue;
+            visited.set(item.id, true);
+            // Find neighbors using existing HNSW infrastructure for efficiency
+            const neighbors = await this._findNeighborsWithinEps(item, itemsWithVectors, eps);
+            if (neighbors.length < minPts) {
+                // Mark as noise (outlier)
+                clusterAssignments.set(item.id, NOISE);
+            }
+            else {
+                // Start new cluster
+                await this._expandCluster(item, neighbors, currentClusterId, eps, minPts, itemsWithVectors, visited, clusterAssignments);
+                currentClusterId++;
+            }
+        }
+        // Convert DBSCAN results to SemanticCluster format
+        const clusters = [];
+        const clusterGroups = new Map();
+        const outliers = [];
+        // Group items by cluster assignment
+        for (const [itemId, clusterId] of clusterAssignments) {
+            if (clusterId === NOISE) {
+                outliers.push(itemId);
+            }
+            else {
+                if (!clusterGroups.has(clusterId)) {
+                    clusterGroups.set(clusterId, []);
+                }
+                clusterGroups.get(clusterId).push(itemId);
+            }
+        }
+        // Create SemanticCluster objects
+        for (const [clusterId, memberIds] of clusterGroups) {
+            if (memberIds.length > 0) {
+                const members = itemsWithVectors.filter(item => memberIds.includes(item.id));
+                clusters.push({
+                    id: `dbscan-${clusterId}`,
+                    centroid: await this._calculateCentroidFromItems(members),
+                    members: memberIds,
+                    size: memberIds.length,
+                    confidence: await this._calculateDBSCANClusterConfidence(members, eps),
+                    label: await this._generateClusterLabel(members, 'dbscan'),
+                    metadata: {
+                        clustering: 'dbscan',
+                        clusterId,
+                        eps,
+                        minPts,
+                        isDensityBased: true
+                    }
+                });
+            }
+        }
+        // Handle outliers - optionally create outlier cluster or assign to nearest
+        if (outliers.length > 0 && options.includeOutliers) {
+            const outlierMembers = itemsWithVectors.filter(item => outliers.includes(item.id));
+            clusters.push({
+                id: 'dbscan-outliers',
+                centroid: await this._calculateCentroidFromItems(outlierMembers),
+                members: outliers,
+                size: outliers.length,
+                confidence: 0.1, // Low confidence for outliers
+                label: 'Outliers',
+                metadata: {
+                    clustering: 'dbscan',
+                    isOutlierCluster: true,
+                    eps,
+                    minPts
+                }
+            });
+        }
+        return {
+            clusters,
+            metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'dbscan'),
+            metadata: {
+                totalItems: itemIds.length,
+                clustersFound: clusters.length,
+                averageClusterSize: clusters.reduce((sum, c) => sum + c.size, 0) / clusters.length || 0,
+                outlierCount: outliers.length,
+                eps,
+                minPts,
+                timestamp: new Date()
+            }
+        };
+    }
+    /**
+     * GRAPH COMMUNITY DETECTION: Uses existing verb relationships for clustering
+     */
+    async _performGraphClustering(items, options) {
+        const startTime = performance.now();
+        const itemIds = items || await this._getAllItemIds();
+        if (itemIds.length === 0) {
+            return this._createEmptyResult(startTime, 'graph');
+        }
+        // Build graph from existing verb relationships
+        const graph = await this._buildGraphFromVerbs(itemIds, options);
+        // Detect communities using modularity optimization
+        const communities = await this._detectCommunities(graph, options);
+        // Enhance communities with vector similarity for boundary refinement
+        const refinedCommunities = await this._refineCommunitiesWithVectors(communities, options);
+        // Convert to SemanticCluster format with Triple Intelligence labeling
+        const clusters = [];
+        for (let i = 0; i < refinedCommunities.length; i++) {
+            const community = refinedCommunities[i];
+            if (community.members.length > 0) {
+                const members = await this._getItemsWithMetadata(community.members);
+                // Use Triple Intelligence for intelligent cluster labeling
+                const clusterLabel = await this._generateIntelligentClusterLabel(members, 'graph');
+                const clusterCentroid = await this._calculateCentroidFromItems(members);
+                clusters.push({
+                    id: `graph-${i}`,
+                    centroid: clusterCentroid,
+                    members: community.members,
+                    size: community.members.length,
+                    confidence: community.modularity || 0.7,
+                    label: clusterLabel,
+                    metadata: {
+                        clustering: 'graph',
+                        communityId: i,
+                        modularity: community.modularity,
+                        graphDensity: community.density,
+                        strongestConnections: community.strongestConnections
+                    }
+                });
+            }
+        }
+        return {
+            clusters,
+            metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'graph'),
+            metadata: {
+                totalItems: itemIds.length,
+                clustersFound: clusters.length,
+                averageClusterSize: clusters.reduce((sum, c) => sum + c.size, 0) / clusters.length || 0,
+                averageModularity: clusters.reduce((sum, c) => sum + (c.metadata?.modularity || 0), 0) / clusters.length || 0,
+                timestamp: new Date()
+            }
+        };
+    }
+    /**
+     * MULTI-MODAL FUSION: Combines vector + graph + semantic + Triple Intelligence
+     */
+    async _performMultiModalClustering(items, options) {
+        const startTime = performance.now();
+        const itemIds = items || await this._getAllItemIds();
+        if (itemIds.length === 0) {
+            return this._createEmptyResult(startTime, 'multimodal');
+        }
+        // Run multiple clustering algorithms in parallel
+        const [vectorClusters, graphClusters, semanticClusters] = await Promise.all([
+            this._performHierarchicalClustering(itemIds, { ...options, algorithm: 'hierarchical' }),
+            this._performGraphClustering(itemIds, { ...options, algorithm: 'graph' }),
+            this._performSemanticClustering(itemIds, { ...options, algorithm: 'semantic' })
+        ]);
+        // Fuse results using intelligent consensus with Triple Intelligence
+        const fusedClusters = await this._fuseClusteringResultsWithTripleIntelligence([vectorClusters.clusters, graphClusters.clusters, semanticClusters.clusters], options);
+        return {
+            clusters: fusedClusters,
+            metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'multimodal'),
+            metadata: {
+                totalItems: itemIds.length,
+                clustersFound: fusedClusters.length,
+                averageClusterSize: fusedClusters.reduce((sum, c) => sum + c.size, 0) / fusedClusters.length || 0,
+                fusionMethod: 'triple_intelligence_consensus',
+                componentAlgorithms: ['hierarchical', 'graph', 'semantic'],
+                timestamp: new Date()
+            }
+        };
+    }
+    /**
+     * SAMPLED CLUSTERING: For very large datasets using intelligent sampling
+     */
+    async _performSampledClustering(items, options) {
+        const startTime = performance.now();
+        const itemIds = items || await this._getAllItemIds();
+        if (itemIds.length === 0) {
+            return this._createEmptyResult(startTime, 'sampled');
+        }
+        const sampleSize = Math.min(options.sampleSize || 1000, itemIds.length);
+        const strategy = options.strategy || 'diverse';
+        // Intelligent sampling using existing infrastructure
+        const sample = await this._getSampleUsingStrategy(itemIds, sampleSize, strategy);
+        // Cluster the sample using the best algorithm for the sample size
+        const sampleResult = await this._performHierarchicalClustering(sample, {
+            ...options,
+            maxClusters: Math.min(options.maxClusters || 50, Math.ceil(sample.length / 10))
+        });
+        // Project clusters back to full dataset using HNSW neighbors
+        const projectedClusters = await this._projectClustersToFullDataset(sampleResult.clusters, itemIds, sample);
+        return {
+            clusters: projectedClusters,
+            metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'sampled'),
+            metadata: {
+                totalItems: itemIds.length,
+                sampleSize: sample.length,
+                samplingStrategy: strategy,
+                clustersFound: projectedClusters.length,
+                averageClusterSize: projectedClusters.reduce((sum, c) => sum + c.size, 0) / projectedClusters.length || 0,
+                timestamp: new Date()
+            }
+        };
+    }
+    // Similarity implementation methods
+    async _similarityById(id1, id2, options) {
+        // Get vectors for both items
+        const item1 = await this.brain.getNoun(id1);
+        const item2 = await this.brain.getNoun(id2);
+        if (!item1 || !item2) {
+            return 0;
+        }
+        return this._similarityByVector(item1.vector, item2.vector, options);
+    }
+    async _similarityByVector(v1, v2, options) {
+        const metric = options.metric || this.config.similarityMetric || 'cosine';
+        let score = 0;
+        switch (metric) {
+            case 'cosine':
+                score = 1 - cosineDistance(v1, v2);
+                break;
+            case 'euclidean':
+                score = 1 / (1 + euclideanDistance(v1, v2));
+                break;
+            case 'manhattan':
+                score = 1 / (1 + this._manhattanDistance(v1, v2));
+                break;
+            default:
+                score = 1 - cosineDistance(v1, v2);
+        }
+        if (options.detailed) {
+            return {
+                score: options.normalized !== false ? Math.max(0, Math.min(1, score)) : score,
+                confidence: this._calculateConfidence(score, v1, v2),
+                explanation: this._generateSimilarityExplanation(score, metric),
+                metric
+            };
+        }
+        return options.normalized !== false ? Math.max(0, Math.min(1, score)) : score;
+    }
+    async _similarityByText(text1, text2, options) {
+        // Convert text to vectors using brain's embedding function
+        const vector1 = await this.brain.embed(text1);
+        const vector2 = await this.brain.embed(text2);
+        return this._similarityByVector(vector1, vector2, options);
+    }
+    // Utility methods for internal operations
+    _isId(value) {
+        return typeof value === 'string' &&
+            (value.length === 36 && value.includes('-')) || // UUID-like
+            (value.length > 10 && !value.includes(' ')); // ID-like string
+    }
+    _isVector(value) {
+        return Array.isArray(value) &&
+            value.length > 0 &&
+            typeof value[0] === 'number';
+    }
+    async _convertToVector(input) {
+        if (this._isVector(input)) {
+            return input;
+        }
+        else if (this._isId(input)) {
+            const item = await this.brain.getNoun(input);
+            return item?.vector || [];
+        }
+        else if (typeof input === 'string') {
+            return await this.brain.embed(input);
+        }
+        else {
+            throw new Error(`Cannot convert input to vector: ${typeof input}`);
+        }
+    }
+    _createSimilarityKey(a, b, options) {
+        const aKey = typeof a === 'object' ? JSON.stringify(a).substring(0, 50) : String(a);
+        const bKey = typeof b === 'object' ? JSON.stringify(b).substring(0, 50) : String(b);
+        return `${aKey}|${bKey}|${JSON.stringify(options)}`;
+    }
+    _createClusteringKey(items, options) {
+        const itemsKey = items ? [...items].sort().join(',') : 'all';
+        return `clustering:${itemsKey}:${JSON.stringify(options)}`;
+    }
+    _cacheResult(key, result, cache) {
+        if (cache.size >= (this.config.cacheSize || 1000)) {
+            // Remove oldest entries (simple LRU)
+            const firstKey = cache.keys().next().value;
+            if (firstKey)
+                cache.delete(firstKey);
+        }
+        cache.set(key, result);
+    }
+    _trackPerformance(operation, startTime, itemCount, algorithm) {
+        if (!this.config.performanceTracking)
+            return;
+        const metrics = {
+            executionTime: performance.now() - startTime,
+            memoryUsed: 0, // Would implement actual memory tracking
+            itemsProcessed: itemCount,
+            cacheHits: 0, // Would track actual cache hits
+            cacheMisses: 0, // Would track actual cache misses
+            algorithm
+        };
+        if (!this.performanceMetrics.has(operation)) {
+            this.performanceMetrics.set(operation, []);
+        }
+        this.performanceMetrics.get(operation).push(metrics);
+    }
+    _createPerformanceMetrics(startTime, itemCount, algorithm) {
+        return {
+            executionTime: performance.now() - startTime,
+            memoryUsed: 0,
+            itemsProcessed: itemCount,
+            cacheHits: 0,
+            cacheMisses: 0,
+            algorithm
+        };
+    }
+    _initializeCleanupTimer() {
+        // Periodically clean up caches to prevent memory leaks
+        setInterval(() => {
+            if (this.similarityCache.size > (this.config.cacheSize || 1000)) {
+                this.similarityCache.clear();
+            }
+            if (this.clusterCache.size > (this.config.cacheSize || 1000)) {
+                this.clusterCache.clear();
+            }
+            if (this.hierarchyCache.size > (this.config.cacheSize || 1000)) {
+                this.hierarchyCache.clear();
+            }
+            if (this.neighborsCache.size > (this.config.cacheSize || 1000)) {
+                this.neighborsCache.clear();
+            }
+        }, 300000); // Clean every 5 minutes
+    }
+    // ===== GRAPH COMMUNITY DETECTION UTILITIES =====
+    /**
+     * Build graph structure from existing verb relationships
+     */
+    async _buildGraphFromVerbs(itemIds, options) {
+        const nodes = new Set(itemIds);
+        const edges = new Map();
+        const verbWeights = new Map();
+        // Initialize verb relationship weights
+        const relationshipWeights = {
+            'creates': 1.0,
+            'partOf': 0.9,
+            'contains': 0.9,
+            'relatedTo': 0.7,
+            'references': 0.6,
+            'causes': 0.8,
+            'dependsOn': 0.8,
+            'memberOf': 0.9,
+            'worksWith': 0.7,
+            'communicates': 0.6
+        };
+        // Get all verbs connecting the items
+        for (const sourceId of itemIds) {
+            const sourceVerbs = await this.brain.getVerbsForNoun(sourceId);
+            for (const verb of sourceVerbs) {
+                const targetId = verb.target;
+                if (nodes.has(targetId) && sourceId !== targetId) {
+                    // Initialize edge map if needed
+                    if (!edges.has(sourceId)) {
+                        edges.set(sourceId, new Map());
+                    }
+                    // Calculate edge weight from verb type and metadata
+                    const verbType = verb.verb;
+                    const baseWeight = relationshipWeights[verbType] || 0.5;
+                    const confidenceWeight = verb.confidence || 1.0;
+                    const weight = baseWeight * confidenceWeight;
+                    // Add or strengthen edge
+                    const currentWeight = edges.get(sourceId)?.get(targetId) || 0;
+                    edges.get(sourceId).set(targetId, Math.min(currentWeight + weight, 1.0));
+                    // Make graph undirected by adding reverse edge
+                    if (!edges.has(targetId)) {
+                        edges.set(targetId, new Map());
+                    }
+                    const reverseWeight = edges.get(targetId)?.get(sourceId) || 0;
+                    edges.get(targetId).set(sourceId, Math.min(reverseWeight + weight, 1.0));
+                }
+            }
+        }
+        return {
+            nodes: Array.from(nodes),
+            edges,
+            nodeCount: nodes.size,
+            edgeCount: Array.from(edges.values()).reduce((sum, edgeMap) => sum + edgeMap.size, 0) / 2 // Undirected
+        };
+    }
+    /**
+     * Detect communities using Louvain modularity optimization
+     */
+    async _detectCommunities(graph, options) {
+        const { nodes, edges } = graph;
+        // Initialize each node as its own community
+        const communities = new Map();
+        nodes.forEach((node, index) => communities.set(node, index));
+        const totalWeight = this._calculateTotalWeight(edges);
+        let improved = true;
+        let iteration = 0;
+        const maxIterations = 50;
+        // Louvain algorithm: iteratively move nodes to communities that maximize modularity
+        while (improved && iteration < maxIterations) {
+            improved = false;
+            iteration++;
+            for (const node of nodes) {
+                const currentCommunity = communities.get(node);
+                let bestCommunity = currentCommunity;
+                let bestGain = 0;
+                // Consider neighboring communities
+                const neighborCommunities = this._getNeighborCommunities(node, edges, communities);
+                for (const neighborCommunity of neighborCommunities) {
+                    if (neighborCommunity !== currentCommunity) {
+                        const gain = this._calculateModularityGain(node, currentCommunity, neighborCommunity, edges, communities, totalWeight);
+                        if (gain > bestGain) {
+                            bestGain = gain;
+                            bestCommunity = neighborCommunity;
+                        }
+                    }
+                }
+                // Move node if beneficial
+                if (bestCommunity !== currentCommunity) {
+                    communities.set(node, bestCommunity);
+                    improved = true;
+                }
+            }
+        }
+        // Group nodes by final community assignment
+        const communityGroups = new Map();
+        for (const [node, communityId] of communities) {
+            if (!communityGroups.has(communityId)) {
+                communityGroups.set(communityId, []);
+            }
+            communityGroups.get(communityId).push(node);
+        }
+        // Convert to Community objects with metadata
+        const result = [];
+        for (const [communityId, members] of communityGroups) {
+            if (members.length >= (options.minClusterSize || 2)) {
+                const modularity = this._calculateCommunityModularity(members, edges, totalWeight);
+                const density = this._calculateCommunityDensity(members, edges);
+                const strongestConnections = this._findStrongestConnections(members, edges, 3);
+                result.push({
+                    id: communityId,
+                    members,
+                    modularity,
+                    density,
+                    strongestConnections
+                });
+            }
+        }
+        return result;
+    }
+    /**
+     * Refine community boundaries using vector similarity
+     */
+    async _refineCommunitiesWithVectors(communities, options) {
+        const refined = [];
+        for (const community of communities) {
+            const membersWithVectors = await this._getItemsWithVectors(community.members);
+            // Check if community is coherent in vector space
+            const vectorCoherence = await this._calculateVectorCoherence(membersWithVectors);
+            if (vectorCoherence > 0.3) {
+                // Community is coherent, keep as is
+                refined.push(community);
+            }
+            else {
+                // Split community using vector-based sub-clustering
+                const subClusters = await this._performHierarchicalClustering(community.members, { ...options, maxClusters: Math.ceil(community.members.length / 5) });
+                // Convert sub-clusters to communities
+                for (let i = 0; i < subClusters.clusters.length; i++) {
+                    const subCluster = subClusters.clusters[i];
+                    refined.push({
+                        id: community.id * 1000 + i, // Unique sub-community ID
+                        members: subCluster.members,
+                        modularity: community.modularity * 0.8, // Slightly lower modularity for sub-communities
+                        density: community.density,
+                        strongestConnections: []
+                    });
+                }
+            }
+        }
+        return refined;
+    }
+    // ===== SEMANTIC CLUSTERING UTILITIES =====
+    /**
+     * Get items with their metadata including noun types
+     */
+    async _getItemsWithMetadata(itemIds) {
+        const items = await Promise.all(itemIds.map(async (id) => {
+            const noun = await this.brain.getNoun(id);
+            return {
+                id,
+                vector: noun?.vector || [],
+                metadata: noun?.data || {},
+                nounType: noun?.noun || 'concept',
+                label: noun?.label || id,
+                data: noun?.data
+            };
+        }));
+        return items.filter(item => item.vector.length > 0);
+    }
+    /**
+     * Group items by their semantic noun types
+     */
+    _groupBySemanticType(items) {
+        const groups = new Map();
+        for (const item of items) {
+            const type = item.nounType;
+            if (!groups.has(type)) {
+                groups.set(type, []);
+            }
+            groups.get(type).push(item);
+        }
+        return groups;
+    }
+    // Placeholder implementations for complex operations
+    async _getAllItemIds() {
+        // Get all noun IDs from the brain
+        const stats = await this.brain.getStatistics();
+        if (!stats.totalNodes || stats.totalNodes === 0) {
+            return [];
+        }
+        // Use a simple approach: get recent items or sample
+        // In practice, this could be optimized with pagination
+        const items = await this.brain.getRecent(Math.min(stats.totalNodes, 10000));
+        return items.map((item) => item.id);
+    }
+    async _getTotalItemCount() {
+        const stats = await this.brain.getStatistics();
+        return stats.totalNodes || 0;
+    }
+    // ===== GRAPH ALGORITHM SUPPORTING METHODS =====
+    _calculateTotalWeight(edges) {
+        let total = 0;
+        for (const edgeMap of edges.values()) {
+            for (const weight of edgeMap.values()) {
+                total += weight;
+            }
+        }
+        return total / 2; // Undirected graph, so divide by 2
+    }
+    _getNeighborCommunities(node, edges, communities) {
+        const neighborCommunities = new Set();
+        const nodeEdges = edges.get(node);
+        if (nodeEdges) {
+            for (const neighbor of nodeEdges.keys()) {
+                const neighborCommunity = communities.get(neighbor);
+                if (neighborCommunity !== undefined) {
+                    neighborCommunities.add(neighborCommunity);
+                }
+            }
+        }
+        return neighborCommunities;
+    }
+    _calculateModularityGain(node, oldCommunity, newCommunity, edges, communities, totalWeight) {
+        // Calculate the degree of the node
+        const nodeDegree = this._getNodeDegree(node, edges);
+        // Calculate edges to old and new communities
+        const edgesToOld = this._getEdgesToCommunity(node, oldCommunity, edges, communities);
+        const edgesToNew = this._getEdgesToCommunity(node, newCommunity, edges, communities);
+        // Calculate community weights
+        const oldCommunityWeight = this._getCommunityWeight(oldCommunity, edges, communities);
+        const newCommunityWeight = this._getCommunityWeight(newCommunity, edges, communities);
+        // Modularity gain calculation (simplified)
+        const oldContrib = edgesToOld - (nodeDegree * oldCommunityWeight) / (2 * totalWeight);
+        const newContrib = edgesToNew - (nodeDegree * newCommunityWeight) / (2 * totalWeight);
+        return newContrib - oldContrib;
+    }
+    _getNodeDegree(node, edges) {
+        const nodeEdges = edges.get(node);
+        if (!nodeEdges)
+            return 0;
+        return Array.from(nodeEdges.values()).reduce((sum, weight) => sum + weight, 0);
+    }
+    _getEdgesToCommunity(node, community, edges, communities) {
+        const nodeEdges = edges.get(node);
+        if (!nodeEdges)
+            return 0;
+        let total = 0;
+        for (const [neighbor, weight] of nodeEdges) {
+            if (communities.get(neighbor) === community) {
+                total += weight;
+            }
+        }
+        return total;
+    }
+    _getCommunityWeight(community, edges, communities) {
+        let total = 0;
+        for (const [node, nodeCommunity] of communities) {
+            if (nodeCommunity === community) {
+                total += this._getNodeDegree(node, edges);
+            }
+        }
+        return total;
+    }
+    _calculateCommunityModularity(members, edges, totalWeight) {
+        if (members.length < 2)
+            return 0;
+        let internalWeight = 0;
+        let totalDegree = 0;
+        for (const member of members) {
+            const memberEdges = edges.get(member);
+            if (memberEdges) {
+                totalDegree += Array.from(memberEdges.values()).reduce((sum, w) => sum + w, 0);
+                // Count internal edges
+                for (const [neighbor, weight] of memberEdges) {
+                    if (members.includes(neighbor)) {
+                        internalWeight += weight;
+                    }
+                }
+            }
+        }
+        internalWeight /= 2; // Undirected graph
+        const expectedInternal = (totalDegree * totalDegree) / (4 * totalWeight);
+        return (internalWeight / totalWeight) - expectedInternal / totalWeight;
+    }
+    _calculateCommunityDensity(members, edges) {
+        if (members.length < 2)
+            return 0;
+        let actualEdges = 0;
+        const maxPossibleEdges = (members.length * (members.length - 1)) / 2;
+        for (const member of members) {
+            const memberEdges = edges.get(member);
+            if (memberEdges) {
+                for (const neighbor of memberEdges.keys()) {
+                    if (members.includes(neighbor) && member < neighbor) { // Avoid double counting
+                        actualEdges++;
+                    }
+                }
+            }
+        }
+        return actualEdges / maxPossibleEdges;
+    }
+    _findStrongestConnections(members, edges, limit) {
+        const connections = [];
+        for (const member of members) {
+            const memberEdges = edges.get(member);
+            if (memberEdges) {
+                for (const [neighbor, weight] of memberEdges) {
+                    if (members.includes(neighbor) && member < neighbor) { // Avoid duplicates
+                        connections.push({ from: member, to: neighbor, weight });
+                    }
+                }
+            }
+        }
+        return connections
+            .sort((a, b) => b.weight - a.weight)
+            .slice(0, limit);
+    }
+    // ===== K-MEANS UTILITIES =====
+    /**
+     * Get items with their vector representations
+     */
+    async _getItemsWithVectors(itemIds) {
+        const items = await Promise.all(itemIds.map(async (id) => {
+            const noun = await this.brain.getNoun(id);
+            return {
+                id,
+                vector: noun?.vector || []
+            };
+        }));
+        return items.filter(item => item.vector.length > 0);
+    }
+    /**
+     * Calculate centroid from items using existing distance functions
+     */
+    async _calculateCentroidFromItems(items) {
+        if (items.length === 0)
+            return [];
+        if (items.length === 1)
+            return [...items[0].vector];
+        const dimensions = items[0].vector.length;
+        const centroid = new Array(dimensions).fill(0);
+        for (const item of items) {
+            for (let i = 0; i < dimensions; i++) {
+                centroid[i] += item.vector[i];
+            }
+        }
+        for (let i = 0; i < dimensions; i++) {
+            centroid[i] /= items.length;
+        }
+        return centroid;
+    }
+    /**
+     * Initialize centroids using k-means++ algorithm for better convergence
+     */
+    async _initializeCentroidsKMeansPlusPlus(items, k) {
+        const centroids = [];
+        // Choose first centroid randomly
+        const firstIdx = Math.floor(Math.random() * items.length);
+        centroids.push([...items[firstIdx].vector]);
+        // Choose remaining centroids using k-means++ probability
+        for (let i = 1; i < k; i++) {
+            const distances = items.map(item => {
+                // Find distance to closest existing centroid
+                let minDist = Infinity;
+                for (const centroid of centroids) {
+                    const dist = this._calculateSquaredDistance(item.vector, centroid);
+                    minDist = Math.min(minDist, dist);
+                }
+                return minDist;
+            });
+            // Choose next centroid with probability proportional to squared distance
+            const totalDistance = distances.reduce((sum, d) => sum + d, 0);
+            const target = Math.random() * totalDistance;
+            let cumulative = 0;
+            for (let j = 0; j < distances.length; j++) {
+                cumulative += distances[j];
+                if (cumulative >= target) {
+                    centroids.push([...items[j].vector]);
+                    break;
+                }
+            }
+        }
+        return centroids;
+    }
+    /**
+     * Assign points to nearest centroids using existing distance functions
+     */
+    async _assignPointsToCentroids(items, centroids) {
+        const assignments = [];
+        for (const item of items) {
+            let bestCentroid = 0;
+            let minDistance = Infinity;
+            for (let i = 0; i < centroids.length; i++) {
+                const distance = this._calculateSquaredDistance(item.vector, centroids[i]);
+                if (distance < minDistance) {
+                    minDistance = distance;
+                    bestCentroid = i;
+                }
+            }
+            assignments.push(bestCentroid);
+        }
+        return assignments;
+    }
+    /**
+     * Update centroids based on current assignments
+     */
+    async _updateCentroids(items, assignments, k) {
+        const newCentroids = [];
+        for (let i = 0; i < k; i++) {
+            const clusterItems = items.filter((_, idx) => assignments[idx] === i);
+            if (clusterItems.length > 0) {
+                newCentroids.push(await this._calculateCentroidFromItems(clusterItems));
+            }
+            else {
+                // Keep old centroid if no items assigned
+                newCentroids.push(new Array(items[0].vector.length).fill(0));
+            }
+        }
+        return newCentroids;
+    }
+    /**
+     * Calculate how much assignments have changed between iterations
+     */
+    _calculateAssignmentChangeRate(oldAssignments, newAssignments) {
+        if (oldAssignments.length !== newAssignments.length)
+            return 1.0;
+        let changes = 0;
+        for (let i = 0; i < oldAssignments.length; i++) {
+            if (oldAssignments[i] !== newAssignments[i]) {
+                changes++;
+            }
+        }
+        return changes / oldAssignments.length;
+    }
+    /**
+     * Calculate cluster confidence for k-means clusters
+     */
+    async _calculateKMeansClusterConfidence(clusterItems, centroid) {
+        if (clusterItems.length <= 1)
+            return 1.0;
+        // Calculate average distance to centroid
+        const distances = clusterItems.map(item => this._calculateSquaredDistance(item.vector, centroid));
+        const avgDistance = distances.reduce((sum, d) => sum + d, 0) / distances.length;
+        // Calculate standard deviation
+        const variance = distances.reduce((sum, d) => sum + Math.pow(d - avgDistance, 2), 0) / distances.length;
+        const stdDev = Math.sqrt(variance);
+        // Higher confidence for tighter clusters
+        const tightness = avgDistance > 0 ? Math.max(0, 1 - (stdDev / avgDistance)) : 1.0;
+        return Math.min(1.0, tightness);
+    }
+    // ===== DBSCAN UTILITIES =====
+    /**
+     * Estimate optimal eps parameter using k-nearest neighbor distances
+     */
+    async _estimateOptimalEps(items, minPts) {
+        if (items.length < minPts)
+            return 0.5;
+        // Calculate k-nearest neighbor distances for each point
+        const kDistances = [];
+        for (const item of items) {
+            const distances = [];
+            for (const otherItem of items) {
+                if (item.id !== otherItem.id) {
+                    const distance = Math.sqrt(this._calculateSquaredDistance(item.vector, otherItem.vector));
+                    distances.push(distance);
+                }
+            }
+            distances.sort((a, b) => a - b);
+            // Get k-th nearest neighbor distance (minPts-1 because we exclude self)
+            const kthDistance = distances[Math.min(minPts - 1, distances.length - 1)];
+            kDistances.push(kthDistance);
+        }
+        kDistances.sort((a, b) => a - b);
+        // Use knee point detection - find point with maximum curvature
+        // Simplified approach: use 90th percentile of k-distances
+        const percentileIndex = Math.floor(kDistances.length * 0.9);
+        return kDistances[percentileIndex] || 0.5;
+    }
+    /**
+     * Find neighbors within epsilon distance using efficient vector operations
+     */
+    async _findNeighborsWithinEps(item, allItems, eps) {
+        const neighbors = [];
+        const epsSquared = eps * eps;
+        for (const otherItem of allItems) {
+            if (item.id !== otherItem.id) {
+                const distanceSquared = this._calculateSquaredDistance(item.vector, otherItem.vector);
+                if (distanceSquared <= epsSquared) {
+                    neighbors.push(otherItem);
+                }
+            }
+        }
+        return neighbors;
+    }
+    /**
+     * Expand DBSCAN cluster by adding density-reachable points
+     */
+    async _expandCluster(seedPoint, neighbors, clusterId, eps, minPts, allItems, visited, clusterAssignments) {
+        clusterAssignments.set(seedPoint.id, clusterId);
+        let i = 0;
+        while (i < neighbors.length) {
+            const neighbor = neighbors[i];
+            if (!visited.get(neighbor.id)) {
+                visited.set(neighbor.id, true);
+                const neighborNeighbors = await this._findNeighborsWithinEps(neighbor, allItems, eps);
+                if (neighborNeighbors.length >= minPts) {
+                    // Add new neighbors to the list (union operation)
+                    for (const newNeighbor of neighborNeighbors) {
+                        if (!neighbors.some(n => n.id === newNeighbor.id)) {
+                            neighbors.push(newNeighbor);
+                        }
+                    }
+                }
+            }
+            // If neighbor is not assigned to any cluster, assign to current cluster
+            if (!clusterAssignments.has(neighbor.id)) {
+                clusterAssignments.set(neighbor.id, clusterId);
+            }
+            i++;
+        }
+    }
+    /**
+     * Calculate DBSCAN cluster confidence based on density
+     */
+    async _calculateDBSCANClusterConfidence(clusterItems, eps) {
+        if (clusterItems.length <= 1)
+            return 1.0;
+        // Calculate average density within the cluster
+        let totalNeighborCount = 0;
+        const epsSquared = eps * eps;
+        for (const item of clusterItems) {
+            let neighborCount = 0;
+            for (const otherItem of clusterItems) {
+                if (item !== otherItem) {
+                    const distanceSquared = this._calculateSquaredDistance(item.vector, otherItem.vector);
+                    if (distanceSquared <= epsSquared) {
+                        neighborCount++;
+                    }
+                }
+            }
+            totalNeighborCount += neighborCount;
+        }
+        const avgDensity = totalNeighborCount / clusterItems.length;
+        const maxPossibleDensity = clusterItems.length - 1;
+        return maxPossibleDensity > 0 ? avgDensity / maxPossibleDensity : 1.0;
+    }
+    // ===== VECTOR UTILITIES =====
+    /**
+     * Calculate squared Euclidean distance (more efficient than sqrt)
+     */
+    _calculateSquaredDistance(vec1, vec2) {
+        if (vec1.length !== vec2.length)
+            return Infinity;
+        let sum = 0;
+        for (let i = 0; i < vec1.length; i++) {
+            const diff = vec1[i] - vec2[i];
+            sum += diff * diff;
+        }
+        return sum;
+    }
+    /**
+     * Calculate vector coherence for community refinement
+     */
+    async _calculateVectorCoherence(items) {
+        if (items.length <= 1)
+            return 1.0;
+        const centroid = await this._calculateCentroidFromItems(items);
+        // Calculate average distance to centroid
+        const distances = items.map(item => Math.sqrt(this._calculateSquaredDistance(item.vector, centroid)));
+        const avgDistance = distances.reduce((sum, d) => sum + d, 0) / distances.length;
+        // Calculate cohesion as inverse of average distance (normalized)
+        const maxDistance = Math.sqrt(centroid.length); // Rough normalization
+        return Math.max(0, 1 - (avgDistance / maxDistance));
+    }
+    async _getItemsByField(field) {
+        // Implementation would query items by metadata field
+        return [];
+    }
+    // ===== TRIPLE INTELLIGENCE INTEGRATION =====
+    /**
+     * Generate intelligent cluster labels using Triple Intelligence
+     */
+    async _generateIntelligentClusterLabel(members, algorithm) {
+        if (members.length === 0)
+            return `${algorithm}-cluster`;
+        try {
+            // Lazy load Triple Intelligence if available
+            const TripleIntelligenceEngine = await import('../triple/TripleIntelligence.js')
+                .then(m => m.TripleIntelligenceEngine)
+                .catch(() => null);
+            if (!TripleIntelligenceEngine) {
+                return this._generateClusterLabel(members, algorithm);
+            }
+            const intelligence = new TripleIntelligenceEngine(this.brain);
+            // Extract key features from cluster members
+            const memberData = members.map(m => ({
+                id: m.id,
+                type: m.nounType,
+                label: m.label,
+                data: m.data
+            }));
+            // Use Triple Intelligence to analyze the cluster and generate label
+            const prompt = `Analyze this cluster of ${memberData.length} related items and provide a concise, descriptive label (2-4 words):
+
+Items:
+${memberData.map(item => `- ${item.label || item.id} (${item.type})`).join('\n')}
+
+The items were grouped using ${algorithm} clustering. What is the most appropriate label that captures their common theme or relationship?`;
+            const response = await intelligence.find({
+                like: prompt,
+                limit: 1
+            });
+            // Extract clean label from response
+            const firstResult = response[0];
+            const label = (firstResult?.metadata?.content || firstResult?.id || `${algorithm}-cluster`)
+                .toString()
+                .replace(/^(Label:|Cluster:|Theme:)/i, '')
+                .trim()
+                .replace(/['"]/g, '')
+                .slice(0, 50);
+            return label || `${algorithm}-cluster`;
+        }
+        catch (error) {
+            // Fallback to simple labeling
+            return this._generateClusterLabel(members, algorithm);
+        }
+    }
+    /**
+     * Generate simple cluster labels based on semantic analysis
+     */
+    async _generateClusterLabel(members, algorithm) {
+        if (members.length === 0)
+            return `${algorithm}-cluster`;
+        // Analyze member types and create descriptive label
+        const typeCount = new Map();
+        for (const member of members) {
+            const type = member.nounType || 'unknown';
+            typeCount.set(type, (typeCount.get(type) || 0) + 1);
+        }
+        // Find most common type
+        let dominantType = 'mixed';
+        let maxCount = 0;
+        for (const [type, count] of typeCount) {
+            if (count > maxCount) {
+                maxCount = count;
+                dominantType = type;
+            }
+        }
+        // Generate label based on dominant type and size
+        const size = members.length;
+        const typePercent = Math.round((maxCount / size) * 100);
+        if (typePercent >= 80) {
+            return `${dominantType} group (${size})`;
+        }
+        else if (typePercent >= 60) {
+            return `mostly ${dominantType} (${size})`;
+        }
+        else {
+            const topTypes = Array.from(typeCount.entries())
+                .sort((a, b) => b[1] - a[1])
+                .slice(0, 2)
+                .map(([type]) => type)
+                .join(' & ');
+            return `${topTypes} cluster (${size})`;
+        }
+    }
+    /**
+     * Fuse clustering results using Triple Intelligence consensus
+     */
+    async _fuseClusteringResultsWithTripleIntelligence(clusterSets, options) {
+        if (clusterSets.length === 0)
+            return [];
+        if (clusterSets.length === 1)
+            return clusterSets[0];
+        // Simple weighted fusion if Triple Intelligence is not available
+        const [vectorClusters, graphClusters, semanticClusters] = clusterSets;
+        // Create consensus mapping of items to clusters
+        const itemClusterMapping = new Map();
+        // Collect all cluster assignments
+        const allAlgorithms = ['vector', 'graph', 'semantic'];
+        const algorithmClusters = [vectorClusters, graphClusters, semanticClusters];
+        for (let i = 0; i < algorithmClusters.length; i++) {
+            const algorithm = allAlgorithms[i];
+            const clusters = algorithmClusters[i] || [];
+            for (const cluster of clusters) {
+                for (const memberId of cluster.members) {
+                    if (!itemClusterMapping.has(memberId)) {
+                        itemClusterMapping.set(memberId, []);
+                    }
+                    itemClusterMapping.get(memberId).push({
+                        algorithm,
+                        clusterId: cluster.id,
+                        confidence: cluster.confidence
+                    });
+                }
+            }
+        }
+        // Find consensus clusters - items that appear together in multiple algorithms
+        const consensusClusters = new Map();
+        const processedItems = new Set();
+        for (const [itemId, assignments] of itemClusterMapping) {
+            if (processedItems.has(itemId))
+                continue;
+            // Find all items that consistently cluster with this item
+            const consensusGroup = new Set([itemId]);
+            // Look for items that share clusters with this item across algorithms
+            for (const assignment of assignments) {
+                const sameClusterItems = this._getItemsInCluster(assignment.clusterId, clusterSets);
+                for (const otherItem of sameClusterItems) {
+                    if (!processedItems.has(otherItem) && otherItem !== itemId) {
+                        const otherAssignments = itemClusterMapping.get(otherItem) || [];
+                        // Check if items co-occur in multiple algorithms
+                        const coOccurrences = this._countCoOccurrences(assignments, otherAssignments);
+                        if (coOccurrences >= 2) { // Must appear together in at least 2 algorithms
+                            consensusGroup.add(otherItem);
+                        }
+                    }
+                }
+            }
+            // Mark all items in this consensus group as processed
+            for (const groupItem of consensusGroup) {
+                processedItems.add(groupItem);
+            }
+            if (consensusGroup.size >= (options.minClusterSize || 2)) {
+                const consensusId = `fusion-${consensusClusters.size}`;
+                consensusClusters.set(consensusId, consensusGroup);
+            }
+        }
+        // Convert consensus groups to SemanticCluster objects
+        const fusedClusters = [];
+        for (const [clusterId, memberSet] of consensusClusters) {
+            const members = Array.from(memberSet);
+            const membersWithMetadata = await this._getItemsWithMetadata(members);
+            if (membersWithMetadata.length > 0) {
+                const centroid = await this._calculateCentroidFromItems(membersWithMetadata);
+                const label = await this._generateIntelligentClusterLabel(membersWithMetadata, 'multimodal');
+                // Calculate fusion confidence based on algorithm agreement
+                const avgConfidence = this._calculateFusionConfidence(members, itemClusterMapping);
+                fusedClusters.push({
+                    id: clusterId,
+                    centroid,
+                    members,
+                    size: members.length,
+                    confidence: avgConfidence,
+                    label,
+                    metadata: {
+                        clustering: 'multimodal_fusion',
+                        algorithms: allAlgorithms,
+                        fusionMethod: 'consensus',
+                        agreementLevel: avgConfidence
+                    }
+                });
+            }
+        }
+        return fusedClusters;
+    }
+    /**
+     * Get items in a specific cluster from cluster sets
+     */
+    _getItemsInCluster(clusterId, clusterSets) {
+        for (const clusterSet of clusterSets) {
+            for (const cluster of clusterSet) {
+                if (cluster.id === clusterId) {
+                    return cluster.members;
+                }
+            }
+        }
+        return [];
+    }
+    /**
+     * Count co-occurrences between two sets of assignments
+     */
+    _countCoOccurrences(assignments1, assignments2) {
+        let count = 0;
+        for (const assignment1 of assignments1) {
+            for (const assignment2 of assignments2) {
+                if (assignment1.algorithm === assignment2.algorithm &&
+                    assignment1.clusterId === assignment2.clusterId) {
+                    count++;
+                }
+            }
+        }
+        return count;
+    }
+    /**
+     * Calculate fusion confidence based on algorithm agreement
+     */
+    _calculateFusionConfidence(members, itemClusterMapping) {
+        let totalConfidence = 0;
+        let totalAssignments = 0;
+        for (const member of members) {
+            const assignments = itemClusterMapping.get(member) || [];
+            for (const assignment of assignments) {
+                totalConfidence += assignment.confidence;
+                totalAssignments++;
+            }
+        }
+        return totalAssignments > 0 ? totalConfidence / totalAssignments : 0.5;
+    }
+    // ===== ADDITIONAL UTILITIES =====
+    /**
+     * Generate empty clustering result for edge cases
+     */
+    _createEmptyResult(startTime, algorithm) {
+        return {
+            clusters: [],
+            metrics: this._createPerformanceMetrics(startTime, 0, algorithm),
+            metadata: {
+                totalItems: 0,
+                clustersFound: 0,
+                averageClusterSize: 0,
+                timestamp: new Date()
+            }
+        };
+    }
+    // ===== SAMPLING AND PROJECTION UTILITIES =====
+    /**
+     * Get sample using specified strategy for large dataset clustering
+     */
+    async _getSampleUsingStrategy(itemIds, sampleSize, strategy) {
+        if (itemIds.length <= sampleSize)
+            return itemIds;
+        switch (strategy) {
+            case 'random':
+                return this._getRandomSample(itemIds, sampleSize);
+            case 'diverse':
+                return await this._getDiverseSample(itemIds, sampleSize);
+            case 'recent':
+                return await this._getRecentSample(itemIds, sampleSize);
+            case 'important':
+                return await this._getImportantSample(itemIds, sampleSize);
+            default:
+                return this._getRandomSample(itemIds, sampleSize);
+        }
+    }
+    /**
+     * Random sampling
+     */
+    _getRandomSample(itemIds, sampleSize) {
+        const shuffled = [...itemIds].sort(() => Math.random() - 0.5);
+        return shuffled.slice(0, sampleSize);
+    }
+    /**
+     * Diverse sampling using vector space distribution
+     */
+    async _getDiverseSample(itemIds, sampleSize) {
+        // Get vectors for all items
+        const itemsWithVectors = await this._getItemsWithVectors(itemIds);
+        if (itemsWithVectors.length <= sampleSize) {
+            return itemIds;
+        }
+        // Use k-means++ style selection for diversity
+        const sample = [];
+        // Select first item randomly
+        let remainingItems = [...itemsWithVectors];
+        const firstIdx = Math.floor(Math.random() * remainingItems.length);
+        sample.push(remainingItems[firstIdx].id);
+        remainingItems.splice(firstIdx, 1);
+        // Select remaining items based on maximum distance to already selected items
+        while (sample.length < sampleSize && remainingItems.length > 0) {
+            let maxDistance = -1;
+            let bestIdx = 0;
+            for (let i = 0; i < remainingItems.length; i++) {
+                const item = remainingItems[i];
+                // Find minimum distance to any selected item
+                let minDistanceToSelected = Infinity;
+                for (const selectedId of sample) {
+                    const selectedItem = itemsWithVectors.find(it => it.id === selectedId);
+                    if (selectedItem) {
+                        const distance = Math.sqrt(this._calculateSquaredDistance(item.vector, selectedItem.vector));
+                        minDistanceToSelected = Math.min(minDistanceToSelected, distance);
+                    }
+                }
+                // Select item with maximum minimum distance (most diverse)
+                if (minDistanceToSelected > maxDistance) {
+                    maxDistance = minDistanceToSelected;
+                    bestIdx = i;
+                }
+            }
+            sample.push(remainingItems[bestIdx].id);
+            remainingItems.splice(bestIdx, 1);
+        }
+        return sample;
+    }
+    /**
+     * Recent sampling based on creation time
+     */
+    async _getRecentSample(itemIds, sampleSize) {
+        const items = await Promise.all(itemIds.map(async (id) => {
+            const noun = await this.brain.getNoun(id);
+            return {
+                id,
+                createdAt: noun?.createdAt || new Date(0)
+            };
+        }));
+        // Sort by creation time (most recent first)
+        items.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
+        return items.slice(0, sampleSize).map(item => item.id);
+    }
+    /**
+     * Important sampling based on connection count and metadata
+     */
+    async _getImportantSample(itemIds, sampleSize) {
+        const items = await Promise.all(itemIds.map(async (id) => {
+            const verbs = await this.brain.getVerbsForNoun(id);
+            const noun = await this.brain.getNoun(id);
+            // Calculate importance score
+            const connectionScore = verbs.length;
+            const dataScore = noun?.data ? Object.keys(noun.data).length : 0;
+            const importanceScore = connectionScore * 2 + dataScore;
+            return {
+                id,
+                importance: importanceScore
+            };
+        }));
+        // Sort by importance (highest first)
+        items.sort((a, b) => b.importance - a.importance);
+        return items.slice(0, sampleSize).map(item => item.id);
+    }
+    /**
+     * Project clusters back to full dataset using HNSW neighbors
+     */
+    async _projectClustersToFullDataset(sampleClusters, fullItemIds, sampleIds) {
+        const projectedClusters = [];
+        // Create mapping of items not in sample
+        const remainingItems = fullItemIds.filter(id => !sampleIds.includes(id));
+        // For each sample cluster, find which remaining items should belong to it
+        for (const sampleCluster of sampleClusters) {
+            const projectedMembers = [...sampleCluster.members];
+            // For each remaining item, find its nearest neighbors in the sample
+            for (const itemId of remainingItems) {
+                try {
+                    const neighbors = await this.brain.neural.neighbors(itemId, {
+                        limit: 3,
+                        includeMetadata: false
+                    });
+                    // Check if any of the nearest neighbors belong to this cluster
+                    let belongsToCluster = false;
+                    for (const neighbor of neighbors.neighbors) {
+                        if (sampleCluster.members.includes(neighbor.id) && neighbor.similarity > 0.7) {
+                            belongsToCluster = true;
+                            break;
+                        }
+                    }
+                    if (belongsToCluster) {
+                        projectedMembers.push(itemId);
+                    }
+                }
+                catch (error) {
+                    // Skip items that can't be processed
+                    continue;
+                }
+            }
+            // Create projected cluster
+            if (projectedMembers.length > 0) {
+                const membersWithVectors = await this._getItemsWithVectors(projectedMembers);
+                projectedClusters.push({
+                    ...sampleCluster,
+                    id: `projected-${sampleCluster.id}`,
+                    members: projectedMembers,
+                    size: projectedMembers.length,
+                    centroid: await this._calculateCentroidFromItems(membersWithVectors),
+                    confidence: sampleCluster.confidence * 0.9, // Slightly lower confidence for projection
+                    metadata: {
+                        ...sampleCluster.metadata,
+                        isProjected: true,
+                        originalSampleSize: sampleCluster.size,
+                        projectedSize: projectedMembers.length
+                    }
+                });
+            }
+        }
+        return projectedClusters;
+    }
+    _groupByDomain(items, field) {
+        const groups = new Map();
+        for (const item of items) {
+            const domain = item.metadata?.[field] || 'unknown';
+            if (!groups.has(domain)) {
+                groups.set(domain, []);
+            }
+            groups.get(domain).push(item);
+        }
+        return groups;
+    }
+    _calculateDomainConfidence(cluster, domainItems) {
+        // Calculate how well this cluster represents the domain
+        return 0.8; // Placeholder
+    }
+    async _findCrossDomainMembers(cluster, threshold) {
+        // Find members that might belong to multiple domains
+        return [];
+    }
+    async _findCrossDomainClusters(clusters, threshold) {
+        // Find clusters that span multiple domains
+        return [];
+    }
+    async _getItemsByTimeWindow(timeField, window) {
+        // Implementation would query items within time window
+        return [];
+    }
+    async _calculateTemporalMetrics(cluster, items, timeField) {
+        // Calculate temporal characteristics of the cluster
+        return {
+            trend: 'stable',
+            metrics: {
+                startTime: new Date(),
+                endTime: new Date(),
+                peakTime: new Date(),
+                frequency: 1
+            }
+        };
+    }
+    _mergeOverlappingTemporalClusters(clusters) {
+        // Merge clusters from overlapping time windows
+        return clusters;
+    }
+    _adjustThresholdAdaptively(clusters, currentThreshold) {
+        // Adjust clustering threshold based on results
+        return currentThreshold || 0.6;
+    }
+    async _calculateItemToClusterSimilarity(itemId, cluster) {
+        // Calculate similarity between an item and a cluster centroid
+        return 0.5; // Placeholder
+    }
+    async _recalculateClusterCentroid(cluster) {
+        // Recalculate centroid after adding new members
+        return cluster.centroid;
+    }
+    async _calculateSimilarity(id1, id2) {
+        return await this.similar(id1, id2);
+    }
+    _sortNeighbors(neighbors, sortBy) {
+        switch (sortBy) {
+            case 'similarity':
+                neighbors.sort((a, b) => b.similarity - a.similarity);
+                break;
+            case 'importance':
+                neighbors.sort((a, b) => (b.metadata?.importance || 0) - (a.metadata?.importance || 0));
+                break;
+            case 'recency':
+                neighbors.sort((a, b) => {
+                    const aTime = new Date(a.metadata?.createdAt || 0).getTime();
+                    const bTime = new Date(b.metadata?.createdAt || 0).getTime();
+                    return bTime - aTime;
+                });
+                break;
+        }
+    }
+    async _buildSemanticHierarchy(item, options) {
+        // Build semantic hierarchy around an item
+        return {
+            self: { id: item.id, vector: item.vector, metadata: item.metadata }
+        };
+    }
+    async _detectOutliersClusterBased(threshold, options) {
+        // Detect outliers using cluster-based method
+        return [];
+    }
+    async _detectOutliersIsolation(threshold, options) {
+        // Detect outliers using isolation forest method
+        return [];
+    }
+    async _detectOutliersStatistical(threshold, options) {
+        // Detect outliers using statistical methods
+        return [];
+    }
+    async _generateVisualizationNodes(maxNodes, options) {
+        // Generate nodes for visualization
+        return [];
+    }
+    async _generateVisualizationEdges(nodes, options) {
+        // Generate edges for visualization
+        return [];
+    }
+    async _generateVisualizationClusters(nodes) {
+        // Generate cluster information for visualization
+        return [];
+    }
+    async _applyLayoutAlgorithm(nodes, edges, algorithm, dimensions) {
+        // Apply layout algorithm to position nodes
+        return nodes.map((node, i) => ({
+            ...node,
+            x: Math.random() * 100,
+            y: Math.random() * 100,
+            z: dimensions === 3 ? Math.random() * 100 : undefined
+        }));
+    }
+    _manhattanDistance(v1, v2) {
+        let sum = 0;
+        for (let i = 0; i < v1.length; i++) {
+            sum += Math.abs(v1[i] - v2[i]);
+        }
+        return sum;
+    }
+    _calculateConfidence(score, v1, v2) {
+        // Calculate confidence based on vector magnitudes and score
+        return Math.min(1, score + 0.1);
+    }
+    _generateSimilarityExplanation(score, metric) {
+        if (score > 0.9)
+            return `Very high similarity using ${metric} distance`;
+        if (score > 0.7)
+            return `High similarity using ${metric} distance`;
+        if (score > 0.5)
+            return `Moderate similarity using ${metric} distance`;
+        if (score > 0.3)
+            return `Low similarity using ${metric} distance`;
+        return `Very low similarity using ${metric} distance`;
+    }
+    // ===== PUBLIC API: UTILITY & STATUS =====
+    /**
+     * Get performance metrics for monitoring
+     */
+    getPerformanceMetrics(operation) {
+        if (operation) {
+            return this.performanceMetrics.get(operation) || [];
+        }
+        return this.performanceMetrics;
+    }
+    /**
+     * Clear all caches
+     */
+    clearCaches() {
+        this.similarityCache.clear();
+        this.clusterCache.clear();
+        this.hierarchyCache.clear();
+        this.neighborsCache.clear();
+    }
+    /**
+     * Get cache statistics
+     */
+    getCacheStats() {
+        const maxSize = this.config.cacheSize || 1000;
+        return {
+            similarity: { size: this.similarityCache.size, maxSize },
+            clustering: { size: this.clusterCache.size, maxSize },
+            hierarchy: { size: this.hierarchyCache.size, maxSize },
+            neighbors: { size: this.neighborsCache.size, maxSize }
+        };
+    }
+    // ===== MISSING HELPER METHODS =====
+    /**
+     * Analyze data characteristics for algorithm selection
+     */
+    async _analyzeDataCharacteristics(itemIds) {
+        const size = itemIds.length;
+        const items = await this._getItemsWithMetadata(itemIds.slice(0, Math.min(100, size)));
+        const dimensionality = items.length > 0 ? items[0].vector.length : 0;
+        // Calculate graph density by sampling verb relationships
+        let connectionCount = 0;
+        const sampleSize = Math.min(50, itemIds.length);
+        for (let i = 0; i < sampleSize; i++) {
+            try {
+                const verbs = await this.brain.getVerbsForNoun(itemIds[i]);
+                connectionCount += verbs.length;
+            }
+            catch (error) {
+                // Skip items that can't be processed
+                continue;
+            }
+        }
+        const graphDensity = sampleSize > 0 ? connectionCount / (sampleSize * sampleSize) : 0;
+        // Calculate type distribution
+        const typeDistribution = {};
+        for (const item of items) {
+            const type = item.nounType;
+            typeDistribution[type] = (typeDistribution[type] || 0) + 1;
+        }
+        return { size, dimensionality, graphDensity, typeDistribution };
+    }
+    /**
+     * Calculate centroid for a group of items
+     */
+    async _calculateGroupCentroid(items) {
+        return this._calculateCentroidFromItems(items);
+    }
+    /**
+     * Cluster within semantic type using vector similarity
+     */
+    async _clusterWithinSemanticType(items, options) {
+        if (items.length <= 2) {
+            return [{
+                    id: `semantic-single-${items[0]?.nounType || 'unknown'}`,
+                    centroid: await this._calculateCentroidFromItems(items),
+                    members: items.map(item => item.id),
+                    size: items.length,
+                    confidence: 1.0,
+                    label: `${items[0]?.nounType || 'unknown'} group`,
+                    metadata: { clustering: 'semantic', nounType: items[0]?.nounType }
+                }];
+        }
+        // Use hierarchical clustering for within-type clustering
+        const result = await this._performHierarchicalClustering(items.map(item => item.id), { ...options, maxClusters: Math.min(Math.ceil(items.length / 3), 10) });
+        return result.clusters;
+    }
+    /**
+     * Find cross-type connections via verbs
+     */
+    async _findCrossTypeConnections(typeGroups, _options) {
+        const connections = [];
+        // Convert Map to array for compatibility
+        const typeGroupsArray = Array.from(typeGroups.entries());
+        for (const [fromType, fromItems] of typeGroupsArray) {
+            for (const [toType, toItems] of typeGroupsArray) {
+                if (fromType !== toType) {
+                    for (const fromItem of fromItems.slice(0, 10)) { // Sample to avoid N^2
+                        try {
+                            const verbs = await this.brain.getVerbsForNoun(fromItem.id);
+                            for (const verb of verbs) {
+                                const toItem = toItems.find(item => item.id === verb.target);
+                                if (toItem) {
+                                    connections.push({
+                                        from: fromItem.id,
+                                        to: toItem.id,
+                                        strength: verb.confidence || 0.7
+                                    });
+                                }
+                            }
+                        }
+                        catch (error) {
+                            // Skip items that can't be processed
+                            continue;
+                        }
+                    }
+                }
+            }
+        }
+        return connections.filter(conn => conn.strength > 0.5);
+    }
+    /**
+     * Merge semantic clusters based on connections
+     */
+    async _mergeSemanticClusters(clusters, connections) {
+        // Simple merging based on strong connections
+        const merged = [...clusters];
+        for (const connection of connections) {
+            if (connection.strength > 0.8) {
+                const fromCluster = merged.find(c => c.members.includes(connection.from));
+                const toCluster = merged.find(c => c.members.includes(connection.to));
+                if (fromCluster && toCluster && fromCluster !== toCluster) {
+                    // Merge clusters
+                    fromCluster.members = [...fromCluster.members, ...toCluster.members];
+                    fromCluster.size = fromCluster.members.length;
+                    fromCluster.label = `merged ${fromCluster.label}`;
+                    // Remove merged cluster
+                    const index = merged.indexOf(toCluster);
+                    if (index > -1)
+                        merged.splice(index, 1);
+                }
+            }
+        }
+        return merged;
+    }
+    /**
+     * Get optimal clustering level for HNSW
+     */
+    _getOptimalClusteringLevel(totalItems) {
+        if (totalItems < 100)
+            return 0;
+        if (totalItems < 1000)
+            return 1;
+        if (totalItems < 10000)
+            return 2;
+        return 3;
+    }
+    /**
+     * Get nodes at HNSW level
+     */
+    async _getHNSWLevelNodes(level) {
+        // This would use the HNSW index to get nodes at specified level
+        // For now, return a sample of all items
+        const allItems = await this._getAllItemIds();
+        const sampleSize = Math.max(10, Math.floor(allItems.length / Math.pow(2, level + 1)));
+        return this._getRandomSample(allItems, sampleSize);
+    }
+    /**
+     * Find cluster members using HNSW neighbors
+     */
+    async _findClusterMembers(levelNode, _allItems, threshold) {
+        try {
+            const neighbors = await this.brain.neural.neighbors(levelNode, {
+                limit: Math.min(50, Math.floor(_allItems.length / 10)),
+                minSimilarity: threshold
+            });
+            return [levelNode, ...neighbors.neighbors.map((n) => n.id)];
+        }
+        catch (error) {
+            return [levelNode];
+        }
+    }
+    /**
+     * Calculate hierarchical clustering confidence
+     */
+    async _calculateHierarchicalConfidence(members) {
+        if (members.length <= 1)
+            return 1.0;
+        const items = await this._getItemsWithVectors(members);
+        const coherence = await this._calculateVectorCoherence(items);
+        return coherence;
+    }
+    /**
+     * Assign unassigned items to nearest clusters
+     */
+    async _assignUnassignedItems(unassigned, clusters) {
+        for (const itemId of unassigned) {
+            if (clusters.length === 0)
+                break;
+            try {
+                const noun = await this.brain.getNoun(itemId);
+                const itemVector = noun?.vector || [];
+                if (itemVector.length === 0)
+                    continue;
+                let bestCluster = clusters[0];
+                let minDistance = Infinity;
+                for (const cluster of clusters) {
+                    const distance = Math.sqrt(this._calculateSquaredDistance(itemVector, cluster.centroid));
+                    if (distance < minDistance) {
+                        minDistance = distance;
+                        bestCluster = cluster;
+                    }
+                }
+                bestCluster.members.push(itemId);
+                bestCluster.size++;
+            }
+            catch (error) {
+                // Skip items that can't be processed
+                continue;
+            }
+        }
+    }
+}
+//# sourceMappingURL=improvedNeuralAPI.js.map