@soulcraft/brainy 3.20.4 → 3.21.0

package/dist/neural/entityExtractor.js

@@ -1,14 +1,18 @@
 /**
  * Neural Entity Extractor using Brainy's NounTypes
  * Uses embeddings and similarity matching for accurate type detection
+ *
+ * PRODUCTION-READY with caching support
  */
 import { NounType } from '../types/graphTypes.js';
+import { EntityExtractionCache, generateFileCacheKey, generateContentCacheKey, computeContentHash } from './entityExtractionCache.js';
 export class NeuralEntityExtractor {
-    constructor(brain) {
+    constructor(brain, cacheOptions) {
         // Type embeddings for similarity matching
         this.typeEmbeddings = new Map();
         this.initialized = false;
         this.brain = brain;
+        this.cache = new EntityExtractionCache(cacheOptions);
     }
     /**
      * Initialize type embeddings for neural matching
@@ -60,9 +64,24 @@ export class NeuralEntityExtractor {
     }
     /**
      * Extract entities from text using neural matching
+     * Now with caching support for performance
      */
     async extract(text, options) {
         await this.initializeTypeEmbeddings();
+        // Check cache if enabled
+        if (options?.cache?.enabled !== false && (options?.path || options?.cache?.invalidateOn === 'hash')) {
+            const cacheKey = options.path
+                ? generateFileCacheKey(options.path)
+                : generateContentCacheKey(text);
+            const cacheOptions = {
+                mtime: options.cache?.mtime,
+                contentHash: !options.path ? computeContentHash(text) : undefined
+            };
+            const cached = this.cache.get(cacheKey, cacheOptions);
+            if (cached) {
+                return cached;
+            }
+        }
         const entities = [];
         const minConfidence = options?.confidence || 0.6;
         const targetTypes = options?.types || Object.values(NounType);
@@ -111,7 +130,19 @@ export class NeuralEntityExtractor {
             }
         }
         // Remove duplicates and overlaps
-        return this.deduplicateEntities(entities);
+        const deduplicatedEntities = this.deduplicateEntities(entities);
+        // Store in cache if enabled
+        if (options?.cache?.enabled !== false && (options?.path || options?.cache?.invalidateOn === 'hash')) {
+            const cacheKey = options.path
+                ? generateFileCacheKey(options.path)
+                : generateContentCacheKey(text);
+            this.cache.set(cacheKey, deduplicatedEntities, {
+                ttl: options.cache?.ttl,
+                mtime: options.cache?.mtime,
+                contentHash: !options.path ? computeContentHash(text) : undefined
+            });
+        }
+        return deduplicatedEntities;
     }
     /**
      * Extract candidate entities using patterns
@@ -312,5 +343,38 @@ export class NeuralEntityExtractor {
         }
         return result;
     }
+    /**
+     * Invalidate cache entry for a specific path or hash
+     */
+    invalidateCache(pathOrHash) {
+        const cacheKey = pathOrHash.includes(':')
+            ? pathOrHash
+            : generateFileCacheKey(pathOrHash);
+        return this.cache.invalidate(cacheKey);
+    }
+    /**
+     * Invalidate all cache entries matching a prefix
+     */
+    invalidateCachePrefix(prefix) {
+        return this.cache.invalidatePrefix(prefix);
+    }
+    /**
+     * Clear all cached entities
+     */
+    clearCache() {
+        this.cache.clear();
+    }
+    /**
+     * Get cache statistics
+     */
+    getCacheStats() {
+        return this.cache.getStats();
+    }
+    /**
+     * Cleanup expired cache entries
+     */
+    cleanupCache() {
+        return this.cache.cleanup();
+    }
 }
 //# sourceMappingURL=entityExtractor.js.map

package/dist/neural/relationshipConfidence.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Relationship Confidence Scoring
+ *
+ * Scores the confidence of detected relationships based on multiple factors:
+ * - Entity proximity in text
+ * - Entity confidence scores
+ * - Pattern matches
+ * - Structural analysis
+ *
+ * PRODUCTION-READY - NO MOCKS, NO STUBS, REAL IMPLEMENTATION
+ */
+import { ExtractedEntity } from './entityExtractor.js';
+import { VerbType } from '../types/graphTypes.js';
+import { RelationEvidence } from '../types/brainy.types.js';
+/**
+ * Detected relationship with confidence
+ */
+export interface DetectedRelationship {
+    sourceEntity: ExtractedEntity;
+    targetEntity: ExtractedEntity;
+    verbType: VerbType;
+    confidence: number;
+    evidence: RelationEvidence;
+}
+/**
+ * Configuration for relationship detection
+ */
+export interface RelationshipDetectionConfig {
+    minConfidence?: number;
+    maxDistance?: number;
+    useProximityBoost?: boolean;
+    usePatternMatching?: boolean;
+    useStructuralAnalysis?: boolean;
+}
+/**
+ * Relationship confidence scorer
+ */
+export declare class RelationshipConfidenceScorer {
+    private config;
+    constructor(config?: RelationshipDetectionConfig);
+    /**
+     * Score a potential relationship between two entities
+     */
+    scoreRelationship(source: ExtractedEntity, target: ExtractedEntity, verbType: VerbType, context: string): {
+        confidence: number;
+        evidence: RelationEvidence;
+    };
+    /**
+     * Calculate proximity boost based on distance between entities
+     */
+    private calculateProximityBoost;
+    /**
+     * Check if entities match a verb pattern
+     */
+    private checkVerbPattern;
+    /**
+     * Analyze structural relationship
+     */
+    private analyzeStructure;
+    /**
+     * Get context text between two entities
+     */
+    private getContextBetween;
+    /**
+     * Detect relationships between a list of entities
+     */
+    detectRelationships(entities: ExtractedEntity[], context: string, verbHints?: VerbType[]): DetectedRelationship[];
+}
+/**
+ * Convenience function to score a single relationship
+ */
+export declare function scoreRelationshipConfidence(source: ExtractedEntity, target: ExtractedEntity, verbType: VerbType, context: string, config?: RelationshipDetectionConfig): {
+    confidence: number;
+    evidence: RelationEvidence;
+};
+/**
+ * Convenience function to detect all relationships in text
+ */
+export declare function detectRelationshipsWithConfidence(entities: ExtractedEntity[], context: string, config?: RelationshipDetectionConfig): DetectedRelationship[];

package/dist/neural/relationshipConfidence.js

@@ -0,0 +1,204 @@
+/**
+ * Relationship Confidence Scoring
+ *
+ * Scores the confidence of detected relationships based on multiple factors:
+ * - Entity proximity in text
+ * - Entity confidence scores
+ * - Pattern matches
+ * - Structural analysis
+ *
+ * PRODUCTION-READY - NO MOCKS, NO STUBS, REAL IMPLEMENTATION
+ */
+import { VerbType } from '../types/graphTypes.js';
+/**
+ * Relationship confidence scorer
+ */
+export class RelationshipConfidenceScorer {
+    constructor(config = {}) {
+        this.config = {
+            minConfidence: config.minConfidence || 0.5,
+            maxDistance: config.maxDistance || 50,
+            useProximityBoost: config.useProximityBoost !== false,
+            usePatternMatching: config.usePatternMatching !== false,
+            useStructuralAnalysis: config.useStructuralAnalysis !== false
+        };
+    }
+    /**
+     * Score a potential relationship between two entities
+     */
+    scoreRelationship(source, target, verbType, context) {
+        let confidence = 0.5; // Base confidence
+        // Evidence tracking
+        const reasoningParts = [];
+        // Factor 1: Proximity boost (closer entities = higher confidence)
+        if (this.config.useProximityBoost) {
+            const proximityBoost = this.calculateProximityBoost(source, target);
+            confidence += proximityBoost;
+            if (proximityBoost > 0) {
+                reasoningParts.push(`Entities are close together (boost: +${proximityBoost.toFixed(2)})`);
+            }
+        }
+        // Factor 2: Entity confidence boost
+        const entityConfidence = (source.confidence + target.confidence) / 2;
+        const entityBoost = (entityConfidence - 0.5) * 0.2; // Scale to 0-0.2
+        confidence *= (1 + entityBoost);
+        if (entityBoost > 0) {
+            reasoningParts.push(`High entity confidence (boost: ${entityBoost.toFixed(2)})`);
+        }
+        // Factor 3: Pattern match boost
+        if (this.config.usePatternMatching) {
+            const patternBoost = this.checkVerbPattern(source, target, verbType, context);
+            confidence += patternBoost;
+            if (patternBoost > 0) {
+                reasoningParts.push(`Matches relationship pattern (boost: +${patternBoost.toFixed(2)})`);
+            }
+        }
+        // Factor 4: Structural boost (same sentence, clause, etc.)
+        if (this.config.useStructuralAnalysis) {
+            const structuralBoost = this.analyzeStructure(source, target, context);
+            confidence += structuralBoost;
+            if (structuralBoost > 0) {
+                reasoningParts.push(`Structural relationship (boost: +${structuralBoost.toFixed(2)})`);
+            }
+        }
+        // Cap confidence at 1.0
+        confidence = Math.min(confidence, 1.0);
+        // Extract source text evidence
+        const start = Math.min(source.position.start, target.position.start);
+        const end = Math.max(source.position.end, target.position.end);
+        const evidence = {
+            sourceText: context.substring(start, end),
+            position: { start, end },
+            method: 'neural',
+            reasoning: reasoningParts.join('; ')
+        };
+        return { confidence, evidence };
+    }
+    /**
+     * Calculate proximity boost based on distance between entities
+     */
+    calculateProximityBoost(source, target) {
+        const distance = Math.abs(source.position.start - target.position.start);
+        if (distance === 0)
+            return 0; // Same position, not meaningful
+        // Very close (< 20 chars): +0.2
+        if (distance < 20)
+            return 0.2;
+        // Close (< 50 chars): +0.1
+        if (distance < 50)
+            return 0.1;
+        // Medium (< 100 chars): +0.05
+        if (distance < 100)
+            return 0.05;
+        // Far (> 100 chars): no boost
+        return 0;
+    }
+    /**
+     * Check if entities match a verb pattern
+     */
+    checkVerbPattern(source, target, verbType, context) {
+        const contextBetween = this.getContextBetween(source, target, context);
+        const contextLower = contextBetween.toLowerCase();
+        // Verb-specific patterns
+        const patterns = {
+            [VerbType.Creates]: ['creates', 'made', 'built', 'developed', 'produces'],
+            [VerbType.Owns]: ['owns', 'belongs to', 'possessed by', 'has'],
+            [VerbType.Contains]: ['contains', 'includes', 'has', 'holds'],
+            [VerbType.Requires]: ['requires', 'needs', 'depends on', 'relies on'],
+            [VerbType.Uses]: ['uses', 'utilizes', 'employs', 'applies'],
+            [VerbType.Supervises]: ['manages', 'oversees', 'supervises', 'controls'],
+            [VerbType.Causes]: ['influences', 'affects', 'impacts', 'shapes', 'causes'],
+            [VerbType.DependsOn]: ['depends on', 'relies on', 'based on'],
+            [VerbType.Modifies]: ['modifies', 'changes', 'alters', 'updates'],
+            [VerbType.References]: ['references', 'cites', 'mentions', 'refers to']
+        };
+        const verbPatterns = patterns[verbType] || [];
+        for (const pattern of verbPatterns) {
+            if (contextLower.includes(pattern)) {
+                return 0.2; // Strong pattern match
+            }
+        }
+        return 0; // No pattern match
+    }
+    /**
+     * Analyze structural relationship
+     */
+    analyzeStructure(source, target, context) {
+        const contextBetween = this.getContextBetween(source, target, context);
+        // Same sentence (no sentence-ending punctuation between them)
+        if (!contextBetween.match(/[.!?]/)) {
+            return 0.1;
+        }
+        // Same paragraph (single newline between them)
+        if (!contextBetween.match(/\n\n/)) {
+            return 0.05;
+        }
+        return 0;
+    }
+    /**
+     * Get context text between two entities
+     */
+    getContextBetween(source, target, context) {
+        const start = Math.min(source.position.end, target.position.end);
+        const end = Math.max(source.position.start, target.position.start);
+        if (start >= end)
+            return '';
+        return context.substring(start, end);
+    }
+    /**
+     * Detect relationships between a list of entities
+     */
+    detectRelationships(entities, context, verbHints) {
+        const relationships = [];
+        const verbs = verbHints || [
+            VerbType.Creates,
+            VerbType.Uses,
+            VerbType.Contains,
+            VerbType.Requires,
+            VerbType.RelatedTo
+        ];
+        // Check all entity pairs
+        for (let i = 0; i < entities.length; i++) {
+            for (let j = i + 1; j < entities.length; j++) {
+                const source = entities[i];
+                const target = entities[j];
+                // Check distance
+                const distance = Math.abs(source.position.start - target.position.start);
+                if (distance > this.config.maxDistance) {
+                    continue; // Too far apart
+                }
+                // Try each verb type
+                for (const verbType of verbs) {
+                    const { confidence, evidence } = this.scoreRelationship(source, target, verbType, context);
+                    if (confidence >= this.config.minConfidence) {
+                        relationships.push({
+                            sourceEntity: source,
+                            targetEntity: target,
+                            verbType,
+                            confidence,
+                            evidence
+                        });
+                    }
+                }
+            }
+        }
+        // Sort by confidence (highest first)
+        relationships.sort((a, b) => b.confidence - a.confidence);
+        return relationships;
+    }
+}
+/**
+ * Convenience function to score a single relationship
+ */
+export function scoreRelationshipConfidence(source, target, verbType, context, config) {
+    const scorer = new RelationshipConfidenceScorer(config);
+    return scorer.scoreRelationship(source, target, verbType, context);
+}
+/**
+ * Convenience function to detect all relationships in text
+ */
+export function detectRelationshipsWithConfidence(entities, context, config) {
+    const scorer = new RelationshipConfidenceScorer(config);
+    return scorer.detectRelationships(entities, context);
+}
+//# sourceMappingURL=relationshipConfidence.js.map

package/dist/storage/baseStorage.js

@@ -240,9 +240,11 @@ export class BaseStorage extends BaseStorageAdapter {
      */
     async getVerbsBySource(sourceId) {
         await this.ensureInitialized();
-        // Use the paginated getVerbs method with source filter
+        // CRITICAL: Fetch ALL verbs for this source, not just first page
+        // This is needed for delete operations to clean up all relationships
         const result = await this.getVerbs({
-            filter: { sourceId }
+            filter: { sourceId },
+            pagination: { limit: Number.MAX_SAFE_INTEGER }
         });
         return result.items;
     }
@@ -251,9 +253,11 @@ export class BaseStorage extends BaseStorageAdapter {
      */
     async getVerbsByTarget(targetId) {
         await this.ensureInitialized();
-        // Use the paginated getVerbs method with target filter
+        // CRITICAL: Fetch ALL verbs for this target, not just first page
+        // This is needed for delete operations to clean up all relationships
         const result = await this.getVerbs({
-            filter: { targetId }
+            filter: { targetId },
+            pagination: { limit: Number.MAX_SAFE_INTEGER }
         });
         return result.items;
     }
@@ -262,9 +266,10 @@ export class BaseStorage extends BaseStorageAdapter {
      */
     async getVerbsByType(type) {
         await this.ensureInitialized();
-        // Use the paginated getVerbs method with type filter
+        // Fetch ALL verbs of this type (no pagination limit)
         const result = await this.getVerbs({
-            filter: { verbType: type }
+            filter: { verbType: type },
+            pagination: { limit: Number.MAX_SAFE_INTEGER }
         });
         return result.items;
     }

package/dist/types/brainy.types.d.ts

@@ -21,6 +21,7 @@ export interface Entity<T = any> {
 }
 /**
  * Relation representation (replaces GraphVerb)
+ * Enhanced with confidence scoring and evidence tracking
  */
 export interface Relation<T = any> {
     id: string;
@@ -32,6 +33,20 @@ export interface Relation<T = any> {
     service?: string;
     createdAt: number;
     updatedAt?: number;
+    confidence?: number;
+    evidence?: RelationEvidence;
+}
+/**
+ * Evidence for why a relationship was detected
+ */
+export interface RelationEvidence {
+    sourceText?: string;
+    position?: {
+        start: number;
+        end: number;
+    };
+    method: 'neural' | 'pattern' | 'structural' | 'explicit';
+    reasoning?: string;
 }
 /**
  * Search result with similarity score
@@ -76,6 +91,7 @@ export interface UpdateParams<T = any> {
 }
 /**
  * Parameters for creating relationships
+ * Enhanced with confidence scoring and evidence tracking
  */
 export interface RelateParams<T = any> {
     from: string;
@@ -85,6 +101,8 @@ export interface RelateParams<T = any> {
     metadata?: T;
     bidirectional?: boolean;
     service?: string;
+    confidence?: number;
+    evidence?: RelationEvidence;
 }
 /**
  * Parameters for updating relationships

package/dist/types/progress.types.d.ts

@@ -0,0 +1,107 @@
+/**
+ * Standardized Progress Reporting
+ *
+ * Provides unified progress tracking across all long-running operations
+ * in Brainy (imports, clustering, large searches, etc.)
+ *
+ * PRODUCTION-READY - NO MOCKS, NO STUBS, REAL IMPLEMENTATION
+ */
+/**
+ * Progress status states
+ */
+export type ProgressStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
+/**
+ * Standardized progress report
+ */
+export interface BrainyProgress<T = any> {
+    status: ProgressStatus;
+    progress: number;
+    message: string;
+    metadata: {
+        itemsProcessed: number;
+        itemsTotal: number;
+        currentItem?: string;
+        estimatedTimeRemaining?: number;
+        startedAt: number;
+        completedAt?: number;
+        throughput?: number;
+    };
+    result?: T;
+    error?: Error;
+}
+/**
+ * Progress tracker with automatic time estimation
+ */
+export declare class ProgressTracker<T = any> {
+    private status;
+    private processed;
+    private total;
+    private startedAt?;
+    private completedAt?;
+    private currentItem?;
+    private result?;
+    private error?;
+    private processingTimes;
+    constructor(total: number);
+    /**
+     * Factory method for creating progress trackers
+     */
+    static create<T>(total: number): ProgressTracker<T>;
+    /**
+     * Start tracking progress
+     */
+    start(): BrainyProgress<T>;
+    /**
+     * Update progress
+     */
+    update(processed: number, currentItem?: string): BrainyProgress<T>;
+    /**
+     * Increment progress by 1
+     */
+    increment(currentItem?: string): BrainyProgress<T>;
+    /**
+     * Mark as completed
+     */
+    complete(result: T): BrainyProgress<T>;
+    /**
+     * Mark as failed
+     */
+    fail(error: Error): BrainyProgress<T>;
+    /**
+     * Mark as cancelled
+     */
+    cancel(): BrainyProgress<T>;
+    /**
+     * Get current progress state
+     */
+    current(): BrainyProgress<T>;
+    /**
+     * Estimate time remaining based on processing history
+     */
+    private estimateTimeRemaining;
+    /**
+     * Calculate current throughput (items/second)
+     */
+    private calculateThroughput;
+    /**
+     * Get progress statistics
+     */
+    getStats(): {
+        status: ProgressStatus;
+        processed: number;
+        total: number;
+        remaining: number;
+        progress: number;
+        elapsed: number;
+        estimatedTotal: number | undefined;
+        throughput: number | undefined;
+    };
+}
+/**
+ * Helper to format time duration
+ */
+export declare function formatDuration(ms: number): string;
+/**
+ * Helper to format progress percentage
+ */
+export declare function formatProgress(progress: BrainyProgress): string;