@rbalchii/anchor-engine 4.7.0

package/docs/archive/search-logic-improvement-plan.md

@@ -0,0 +1,336 @@
+# Search Logic Improvement Plan for ECE_Core
+
+## Problem Statement
+
+The Tag-Walker protocol shows brittleness in natural language query processing:
+- Query "What is the latest state of the ECE" returned 0 results
+- Fallback query "state ECE" returned 42 relevant results
+- NLP parser over-optimizes or filters are too strict
+- System relies on fallback strategies instead of primary semantic matching
+
+## Root Cause Analysis
+
+### 1. Query Intent Mapping Issue
+The system fails to map natural language intent to semantic categories effectively.
+
+### 2. Overly Restrictive Filtering
+First-pass filtering eliminates potentially relevant results too aggressively.
+
+### 3. Semantic Category Matching
+The system doesn't properly recognize semantic relationships in conversational queries.
+
+## Implementation Plan
+
+### Phase 1: Immediate Fixes (Week 1)
+
+#### 1.1 Relax First-Pass Filtering
+**File**: `engine/src/services/search/tag-walker.ts`
+
+```typescript
+// Current implementation may be too restrictive
+const strictFilters = {
+  minRelevance: 0.8,  // TOO HIGH
+  bucketMatchRequired: true,  // TOO RESTRICTIVE
+  tagMatchThreshold: 0.9    // TOO HIGH
+};
+
+// Updated implementation
+const relaxedFilters = {
+  minRelevance: 0.3,  // MORE PERMISSIVE
+  bucketMatchRequired: false,  // ALLOW FLEXIBILITY
+  tagMatchThreshold: 0.5    // MORE INCLUSIVE
+};
+```
+
+#### 1.2 Enhance Natural Language Processing
+**File**: `engine/src/services/nlp/query-parser.ts`
+
+```typescript
+// Add conversational query expansion
+function expandConversationalQuery(query: string): string[] {
+  const expansions: string[] = [];
+
+  // Common conversational patterns
+  const patterns = [
+    { pattern: /what is the (latest|current|recent) (.+)/i, expansion: "$2" },
+    { pattern: /tell me about (.+)/i, expansion: "$1" },
+    { pattern: /how is (.+) doing/i, expansion: "$1" },
+    { pattern: /what's happening with (.+)/i, expansion: "$1" }
+  ];
+
+  for (const p of patterns) {
+    const match = query.match(p.pattern);
+    if (match) {
+      expansions.push(query.replace(p.pattern, p.expansion));
+    }
+  }
+
+  return expansions;
+}
+```
+
+#### 1.3 Improve Query Confidence Scoring
+**File**: `engine/src/services/search/search-service.ts`
+
+```typescript
+// Add confidence-based fallback mechanism
+interface QueryResult {
+  content: string;
+  score: number;
+  confidence: number;  // NEW FIELD
+  strategy: string;
+}
+
+function executeSearch(query: string): QueryResult[] {
+  // Primary search with relaxed filters
+  const primaryResults = executeTagWalkerSearch(query, {
+    strictness: 'relaxed',
+    minConfidence: 0.3  // LOWER THRESHOLD
+  });
+
+  // If primary search yields insufficient results, try expansions
+  if (primaryResults.length < 3) {
+    const expandedQueries = expandConversationalQuery(query);
+    for (const expQuery of expandedQueries) {
+      const expansionResults = executeTagWalkerSearch(expQuery, {
+        strictness: 'balanced',
+        minConfidence: 0.5
+      });
+      primaryResults.push(...expansionResults);
+    }
+  }
+
+  return primaryResults;
+}
+```
+
+### Phase 2: Semantic Enhancement (Week 2)
+
+#### 2.1 Entity Co-occurrence Detection Improvement
+**File**: `engine/src/services/semantic/entity-cooccurrence.ts`
+
+```typescript
+// Enhanced entity co-occurrence detection
+class EntityCooccurrenceDetector {
+  detectRelationships(content: string, entities: string[]): SemanticRelationship[] {
+    const relationships: SemanticRelationship[] = [];
+
+    // Look for relationship indicators between entities
+    const relationshipIndicators = [
+      'and', 'with', 'met', 'told', 'said', 'visited', 'called',
+      'texted', 'about', 'relationship', 'connection', 'interaction'
+    ];
+
+    // Enhanced detection logic
+    for (let i = 0; i < entities.length; i++) {
+      for (let j = i + 1; j < entities.length; j++) {
+        const entity1 = entities[i];
+        const entity2 = entities[j];
+
+        // Check for relationship indicators between entities
+        const pattern = new RegExp(`${entity1}[^.!?]{0,100}${relationshipIndicators.join('|')}[^.!?]{0,100}${entity2}|${entity2}[^.!?]{0,100}${relationshipIndicators.join('|')}[^.!?]{0,100}${entity1}`, 'gi');
+
+        if (content.match(pattern)) {
+          relationships.push({
+            entity1,
+            entity2,
+            type: 'relationship',
+            strength: 0.8
+          });
+        }
+      }
+    }
+
+    return relationships;
+  }
+}
+```
+
+#### 2.2 Semantic Category Mapping
+**File**: `engine/src/services/semantic/category-mapper.ts`
+
+```typescript
+// Enhanced semantic category mapping
+class SemanticCategoryMapper {
+  mapQueryToCategories(query: string): SemanticCategory[] {
+    const categories: SemanticCategory[] = [];
+
+    // Enhanced mapping logic
+    if (this.containsSemanticIndicators(query, ['state', 'status', 'current', 'latest', 'recent'])) {
+      categories.push(SemanticCategory.STATUS);
+    }
+
+    if (this.containsSemanticIndicators(query, ['relationship', 'connection', 'interaction', 'with', 'and'])) {
+      categories.push(SemanticCategory.RELATIONSHIP);
+    }
+
+    if (this.containsSemanticIndicators(query, ['technical', 'code', 'architecture', 'system', 'implementation'])) {
+      categories.push(SemanticCategory.TECHNICAL);
+    }
+
+    // Add context-specific mappings
+    if (query.toLowerCase().includes('ece') || query.toLowerCase().includes('engine')) {
+      categories.push(SemanticCategory.PROJECT);
+    }
+
+    return categories;
+  }
+
+  private containsSemanticIndicators(text: string, indicators: string[]): boolean {
+    const lowerText = text.toLowerCase();
+    return indicators.some(indicator => lowerText.includes(indicator));
+  }
+}
+```
+
+### Phase 3: Adaptive Filtering (Week 3)
+
+#### 3.1 Query-Type Adaptive Filtering
+**File**: `engine/src/services/search/adaptative-filter.ts`
+
+```typescript
+// Adaptive filtering based on query type
+class AdaptiveFilter {
+  applyFilter(query: string, results: SearchMolecule[], queryType: QueryType): SearchMolecule[] {
+    switch (queryType) {
+      case 'conversational':
+        // More permissive for conversational queries
+        return results.filter(r => r.score >= 0.2);
+
+      case 'precise':
+        // More restrictive for precise queries
+        return results.filter(r => r.score >= 0.7);
+
+      case 'exploratory':
+        // Medium restrictiveness for exploratory queries
+        return results.filter(r => r.score >= 0.4);
+
+      default:
+        // Default to balanced filtering
+        return results.filter(r => r.score >= 0.5);
+    }
+  }
+
+  classifyQueryType(query: string): QueryType {
+    // Classify based on linguistic patterns
+    if (query.toLowerCase().startsWith('what') ||
+        query.toLowerCase().startsWith('how') ||
+        query.toLowerCase().startsWith('tell me')) {
+      return 'conversational';
+    }
+
+    if (query.split(' ').length <= 3) {
+      return 'precise';
+    }
+
+    return 'exploratory';
+  }
+}
+```
+
+### Phase 4: Monitoring and Feedback (Week 4)
+
+#### 4.1 Query Success Rate Tracking
+**File**: `engine/src/services/monitoring/query-analytics.ts`
+
+```typescript
+// Query analytics for continuous improvement
+class QueryAnalytics {
+  private successRateTracker: Map<string, { successes: number, failures: number }> = new Map();
+
+  recordQueryOutcome(query: string, resultsCount: number, strategy: string): void {
+    const key = `${strategy}_${query.substring(0, 20)}`;
+    const stats = this.successRateTracker.get(key) || { successes: 0, failures: 0 };
+
+    if (resultsCount > 0) {
+      stats.successes++;
+    } else {
+      stats.failures++;
+    }
+
+    this.successRateTracker.set(key, stats);
+  }
+
+  getSuccessRates(): Map<string, number> {
+    const rates = new Map<string, number>();
+
+    for (const [key, stats] of this.successRateTracker.entries()) {
+      const total = stats.successes + stats.failures;
+      if (total > 0) {
+        rates.set(key, stats.successes / total);
+      }
+    }
+
+    return rates;
+  }
+}
+```
+
+## Implementation Timeline
+
+### Week 1: Immediate Fixes
+- [ ] Relax first-pass filtering parameters
+- [ ] Implement conversational query expansion
+- [ ] Add confidence-based fallback mechanism
+- [ ] Test with sample queries including "What is the latest state of the ECE"
+
+### Week 2: Semantic Enhancement
+- [ ] Improve entity co-occurrence detection
+- [ ] Enhance semantic category mapping
+- [ ] Test relationship narrative discovery
+- [ ] Validate with diverse query types
+
+### Week 3: Adaptive Filtering
+- [ ] Implement query-type classification
+- [ ] Apply adaptive filtering based on query type
+- [ ] Fine-tune thresholds for different query types
+- [ ] Test performance impact
+
+### Week 4: Monitoring and Optimization
+- [ ] Deploy query analytics
+- [ ] Monitor success rates
+- [ ] Iterate based on real-world usage
+- [ ] Document improvements
+
+## Success Metrics
+
+### Primary Metrics
+- **Query Success Rate**: Increase from current rate to >95%
+- **Zero-Result Queries**: Reduce to <2% of total queries
+- **Fallback Usage**: Reduce reliance on fallback strategies by 50%
+
+### Secondary Metrics
+- **Response Time**: Maintain <100ms average response time
+- **Precision/Recall Balance**: Optimize for user satisfaction
+- **User Engagement**: Track query frequency and session duration
+
+## Rollout Strategy
+
+### Phase 1: Internal Testing
+- Deploy changes to development environment
+- Run comprehensive test suite
+- Validate with sample queries including the problematic "What is the latest state of the ECE"
+
+### Phase 2: Staged Rollout
+- Deploy to subset of users
+- Monitor query analytics
+- Collect feedback on search quality
+
+### Phase 3: Full Deployment
+- Deploy to all users
+- Monitor system-wide metrics
+- Continue optimization based on usage patterns
+
+## Risk Mitigation
+
+### Performance Risks
+- Monitor system performance during rollout
+- Maintain performance benchmarks
+- Rollback plan if performance degrades
+
+### Quality Risks
+- Implement A/B testing for search quality
+- Monitor user feedback
+- Maintain fallback mechanisms during transition
+
+This plan addresses the specific issue where natural language queries fail to return results despite relevant content existing in the system. The improvements will make the search more robust while maintaining the high performance achieved through the native module optimizations.

package/docs/archive/search_architecture_diagram.md

@@ -0,0 +1,212 @@
+# Search Architecture Diagram for ECE_Core
+
+## Tag-Walker Protocol Overview
+
+```mermaid
+graph TD
+    subgraph "Query Processing"
+        A[User Query<br/>Natural Language]
+        B[Query Parser<br/>NLP Processing]
+        C[Query Expander<br/>Synonym Mapping]
+    end
+
+    subgraph "Search Phases"
+        D[Phase 1: FTS Search<br/>PGlite GIN Index]
+        E[Phase 2: Tag-Walker<br/>Graph Traversal]
+        F[Phase 3: Context Inflation<br/>Molecular Coordinates]
+    end
+
+    subgraph "Database Layer"
+        G[(PGlite Database)]
+        H[Atoms Table<br/>Content + FTS Index]
+        I[Tags Table<br/>Semantic Relationships]
+        J[Edges Table<br/>Atom Connections]
+        K[Engrams Table<br/>Lexical Sidecar]
+    end
+
+    subgraph "Results Processing"
+        L[Result Scoring<br/>Relevance Ranking]
+        M[Duplicate Filtering<br/>SimHash Deduplication]
+        N[Response Assembly<br/>Context Window Formation]
+    end
+
+    A --> B
+    B --> C
+    C --> D
+    C --> E
+    C --> F
+
+    D --> G
+    E --> G
+    F --> G
+
+    G --> H
+    G --> I
+    G --> J
+    G --> K
+    G --> AP
+
+    H --> L
+    I --> L
+    J --> L
+    K --> L
+    AP --> L
+
+    L --> M
+    M --> N
+```
+
+## Detailed Search Flow
+
+```mermaid
+sequenceDiagram
+    participant UI as User Interface
+    participant API as API Service
+    participant SS as Search Service
+    participant TW as Tag-Walker Protocol
+    participant DB as PGlite Database
+    participant NS as Native Services
+
+    UI->>API: POST /v1/memory/search {query: "revenue optimization", buckets: ["inbox"], max_chars: 20000}
+    API->>SS: executeSearch(query, buckets, maxChars)
+    SS->>TW: processQuery(query)
+    TW->>DB: Execute FTS search on content
+    DB-->>TW: Initial results from GIN index
+    alt No Results Found
+        TW->>DB: Execute fuzzy fallback (OR logic)
+        DB-->>TW: Fallback results
+    end
+    TW->>TW: Apply Tag-Walker graph traversal
+    TW->>DB: Query related atoms via tags/edges
+    DB-->>TW: Graph traversal results
+    TW->>SS: Curated results with provenance
+    SS->>SS: Apply context inflation (Radial Inflation form Disk)
+    SS->>DB: Query atom_positions for byte offsets
+    SS->>NS: Partial file read (fs.readSync)
+    NS-->>SS: Processed results
+    SS-->>API: Formatted response with context window
+    API-->>UI: Search results (limited by token budget)
+```
+
+## Smart Search Protocol (Standard 094)
+
+```mermaid
+graph TD
+    subgraph "Phase 1: Intelligent Parsing"
+        A[Input: "What do we know about revenue optimization summary"]
+        B[POS Tagging & Stopword Removal]
+        C[Output: "revenue optimization summary"]
+    end
+
+    subgraph "Phase 2: Strict Anchor Search"
+        D[GIN Index Query<br/>All terms must match]
+        E[PGlite FTS with 'simple' dictionary]
+        F[Strict AND logic: revenue & optimization & summary]
+    end
+
+    subgraph "Phase 3: Fuzzy Fallback"
+        G[Trigger: Zero results from Phase 2]
+        H[Logical OR query: revenue | optimization | summary]
+        I[Retrieve partial matches]
+    end
+
+    subgraph "Phase 4: Semantic Walk"
+        J[Use seed results as graph nodes]
+        K[Traverse related atoms via shared tags]
+        L[Pull in associated content]
+    end
+
+    A --> B
+    B --> C
+    C --> D
+    D --> E
+    E --> F
+    F -->|Zero Results| G
+    G --> H
+    H --> I
+    F -->|Results Found| J
+    I -->|Results Found| J
+    J --> K
+    K --> L
+```
+
+## Performance Optimization Layers
+
+```mermaid
+graph LR
+    subgraph "Caching Layer"
+        CA[Query Cache<br/>Recent search results]
+        CB[Atom Cache<br/>Frequently accessed atoms]
+        CC[Embedding Cache<br/>Computed embeddings]
+    end
+
+    subgraph "Indexing Layer"
+        DA[GIN Index<br/>Full-Text Search]
+        DB[B-tree Indices<br/>Tags, Buckets, Timestamps]
+        DC[Hash Indices<br/>SimHash for deduplication]
+    end
+
+    subgraph "Processing Layer"
+        EA[Native Modules<br/>C++ Performance]
+        EB[Batch Processing<br/>SIMD Operations]
+        EC[Zero-Copy Operations<br/>std::string_view]
+    end
+
+    CA --> DA
+    CB --> DB
+    CC --> DC
+    DA --> EA
+    DB --> EB
+    DC --> EC
+```
+
+## Agent Integration Search Flow
+
+```mermaid
+sequenceDiagram
+    participant AH as Agent Harness
+    participant API as REST API
+    participant SS as Search Service
+    participant TW as Tag-Walker
+    participant DB as PGlite
+    participant AG as Agent Runtime
+
+    AH->>API: Query with context requirement
+    API->>SS: Retrieve relevant context
+    SS->>TW: Execute Tag-Walker search
+    TW->>DB: Query atoms with semantic matching
+    DB-->>TW: Retrieved context atoms
+    TW-->>SS: Curated results
+    SS-->>API: Formatted context
+    API->>AG: Combine context + agent logic
+    AG-->>API: Processed response
+    API-->>AH: Final response with agent reasoning
+```
+
+## Database Schema for Search
+
+```mermaid
+graph BT
+    subgraph "PGlite Tables"
+        AT[Atoms Table<br/>- id (TEXT)<br/>- content (TEXT)<br/>- source_path (TEXT)<br/>- timestamp (REAL)<br/>- buckets (TEXT[])<br/>- tags (TEXT[])<br/>- epochs (TEXT[])<br/>- simhash (TEXT)<br/>- embedding (TEXT)<br/>- provenance (TEXT)<br/>- payload (JSONB)]
+        
+        TT[Tags Table<br/>- atom_id (TEXT)<br/>- tag (TEXT)<br/>- bucket (TEXT)<br/>PRIMARY KEY (atom_id, tag)]
+        
+        ET[Edges Table<br/>- source_id (TEXT)<br/>- target_id (TEXT)<br/>- relation (TEXT)<br/>- weight (REAL)<br/>PRIMARY KEY (source_id, target_id, relation)]
+        
+        ST[Sources Table<br/>- path (TEXT)<br/>- hash (TEXT)<br/>- total_atoms (INTEGER)<br/>- last_ingest (REAL)<br/>PRIMARY KEY (path)]
+        
+        MT[Molecules Table<br/>- id (TEXT)<br/>- content (TEXT)<br/>- compound_id (TEXT)<br/>- sequence (INTEGER)<br/>- start_byte (INTEGER)<br/>- end_byte (INTEGER)<br/>- type (TEXT)<br/>- numeric_value (REAL)<br/>- numeric_unit (TEXT)<br/>- molecular_signature (TEXT)<br/>- embedding (TEXT)<br/>- timestamp (REAL)]
+        
+        CT[Compounds Table<br/>- id (TEXT)<br/>- compound_body (TEXT)<br/>- path (TEXT)<br/>- timestamp (REAL)<br/>- provenance (TEXT)<br/>- molecular_signature (TEXT)<br/>- atoms (TEXT)<br/>- molecules (TEXT)<br/>- embedding (TEXT)<br/>PRIMARY KEY (id)]
+
+        AP[Atom Positions Table<br/>- compound_id (TEXT)<br/>- atom_label (TEXT)<br/>- byte_offset (INTEGER)<br/>PRIMARY KEY (compound_id, atom_label, byte_offset)]
+    end
+
+    AT -.-> TT
+    AT -.-> ET
+    AT -.-> ST
+    AT -.-> MT
+    AT -.-> CT
+    AT -.-> AP
+```

package/docs/archive/semantic-architecture-guide.md

@@ -0,0 +1,97 @@
+# Semantic Shift Architecture Guide
+
+## Overview
+The ECE Core now implements the Semantic Shift Architecture (Standard 084) that transforms the system from a keyword indexer to a semantic graph with relationship narrative discovery capabilities.
+
+## Key Features
+
+### 1. Semantic Categories
+Instead of granular tags, the system now uses high-level semantic categories:
+- `#Relationship` - Personal and professional connections
+- `#Narrative` - Stories, timelines, and memory sequences  
+- `#Technical` - Code, architecture, and system documentation
+- `#Industry` - External market and domain-specific data
+- `#Location` - Geographic and spatial references
+- `#Emotional` - High sentiment variance content
+- `#Temporal` - Time-based sequences and chronology
+- `#Causal` - Cause-effect relationships
+
+### 2. Entity Co-occurrence Detection
+The system automatically detects when entities appear together in the same semantic molecule:
+- When "Rob" and "Jade" appear together → `#Relationship` tag
+- When person and time reference appear → `#Narrative` tag
+- When technical terms appear → `#Technical` tag
+
+### 3. Relationship Narrative Discovery
+The system can extract relationship narratives by finding entity co-occurrence patterns across your data.
+
+## How to Use
+
+### Search for Relationships
+To search for relationship content, include multiple entities in your query:
+- "Rob and Jade" - Finds content where both entities appear together
+- "Tell me about Rob and Dory" - Returns relationship narratives between these entities
+
+### Use Semantic Categories in Queries
+You can filter by semantic categories using hashtags:
+- "Jade #Relationship" - Find relationship content about Jade
+- "CO2 #Industry" - Find industrial content about CO2
+- "Albuquerque #Location" - Find location-based content about Albuquerque
+
+### Toggle Features
+The UI includes two important toggles:
+- **Save to Graph**: When enabled, saves conversations to the knowledge graph
+- **Use Port 8080**: When enabled, routes API requests to port 8080 for Qwen Code CLI integration
+
+## Architecture Benefits
+
+### Improved Search Relevance
+- Relationship queries return only relationship content
+- Technical queries return only technical content
+- Narrative queries return only story content
+
+### Reduced Noise
+- No more irrelevant matches mixing domains
+- Semantic filtering keeps content types separate
+- Context-first approach eliminates session memory baggage
+
+### Universal Application
+- Same architecture works for personal relationships
+- Same architecture works for industrial data
+- Same architecture works for technical documentation
+
+## Implementation Details
+
+### Semantic Molecule Processing
+Text chunks are processed as semantic molecules with high-level tags instead of granular entity tagging.
+
+### Stateless Contextual Chat
+Each query gets fresh context from ECE search results instead of relying on chat session history.
+
+### Dynamic Taxonomy
+The system can adapt to different domains while maintaining semantic category consistency.
+
+## Examples
+
+### Personal Domain
+```
+Query: "What happened between Rob and Jade?"
+Result: Finds all content where "Rob" and "Jade" appear together, highlighting their relationship narrative.
+```
+
+### Industrial Domain
+```
+Query: "How is CO2 related to sequestration?"
+Result: Finds content where "CO2" and "sequestration" appear together, highlighting industrial relationships.
+```
+
+### Technical Domain
+```
+Query: "Explain the relationship between CozoDB and ECE?"
+Result: Finds technical content where these concepts appear together, highlighting system architecture relationships.
+```
+
+## Standards Compliance
+- Implements Standard 084: Semantic Shift Architecture
+- Maintains backward compatibility with existing functionality
+- Preserves the "Sovereign" nature of the context engine