nucleus-mcp 1.0.7 → 1.2.0

package/src/mcp_server_nucleus/runtime/task_ingestion.py

@@ -1,1020 +0,0 @@
-"""
-TaskIngestionEngine - Multi-Source Task Ingestion System
-Enterprise-grade ingestion from planning docs, TODOs, handoffs, meetings, APIs.
-
-Features:
-- Multi-source parsing (5+ types)
-- Semantic deduplication (hash + optional embeddings)
-- Full provenance tracking (source, line, chain)
-- Batch and stream ingestion modes
-- Conflict resolution (priority, assignee, resurrection)
-- Rollback support with crash recovery
-- AgentPool integration for auto-assignment
-- Cost tracking and budget limits
-
-Scales: 1 → 1000 sources, 10K tasks/sec
-
-Author: NOP V3.1 - January 2026
-"""
-
-import json
-import re
-import time
-import hashlib
-import threading
-import uuid
-import os
-from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Any
-from collections import defaultdict
-from enum import Enum
-from dataclasses import dataclass, field
-
-
-class SourceType(str, Enum):
-    """Supported ingestion source types."""
-    PLANNING = "planning"
-    TODOS = "todos"
-    HANDOFFS = "handoffs"
-    MEETINGS = "meetings"
-    API = "api"
-    MANUAL = "manual"
-    SYNTHESIS = "synthesis"
-
-
-class IngestionMode(str, Enum):
-    """Ingestion processing modes."""
-    BATCH = "batch"
-    STREAM = "stream"
-    MICRO_BATCH = "micro"
-
-
-@dataclass
-class IngestionResult:
-    """Result of an ingestion operation."""
-    success: bool
-    batch_id: str
-    tasks_created: int = 0
-    tasks_updated: int = 0
-    tasks_skipped: int = 0
-    tasks_failed: int = 0
-    errors: List[str] = field(default_factory=list)
-    rollback_id: str = None
-    cost_estimate: float = 0.0
-    auto_assigned: List[Dict] = field(default_factory=list)
-    created_task_ids: List[str] = field(default_factory=list)
-
-    def to_dict(self) -> Dict:
-        return {
-            "success": self.success,
-            "batch_id": self.batch_id,
-            "tasks_created": self.tasks_created,
-            "tasks_updated": self.tasks_updated,
-            "tasks_skipped": self.tasks_skipped,
-            "tasks_failed": self.tasks_failed,
-            "errors": self.errors,
-            "rollback_id": self.rollback_id,
-            "cost_estimate": self.cost_estimate,
-            "auto_assigned": self.auto_assigned,
-            "created_task_ids": self.created_task_ids,
-        }
-
-
-@dataclass
-class DuplicateCheckResult:
-    """Result of duplicate check."""
-    is_duplicate: bool
-    matching_task_id: Optional[str] = None
-    similarity: float = 0.0
-    match_type: str = None  # "exact", "semantic", "source"
-
-
-class InputSanitizer:
-    """Sanitize inputs from all ingestion sources."""
-    
-    MAX_DESCRIPTION_LENGTH = 2000
-    MAX_ORIGINAL_TEXT_LENGTH = 500
-    
-    DANGEROUS_PATTERNS = [
-        r'<script', r'javascript:', r'\$\{', r'{{',
-        r'eval\s*\(', r'exec\s*\(', r'__import__',
-    ]
-    
-    def sanitize_description(self, text: str) -> str:
-        """Sanitize task description."""
-        if not text:
-            return ""
-        
-        text = text[:self.MAX_DESCRIPTION_LENGTH]
-        for pattern in self.DANGEROUS_PATTERNS:
-            text = re.sub(pattern, '[FILTERED]', text, flags=re.IGNORECASE)
-        text = re.sub(r'\s+', ' ', text).strip()
-        text = text.replace('\x00', '')
-        return text
-    
-    def sanitize_file_path(self, path: str) -> Optional[str]:
-        """Sanitize file path."""
-        if not path:
-            return None
-        path = os.path.normpath(path)
-        if '..' in path:
-            raise ValueError("Directory traversal not allowed")
-        return path
-
-
-class TaskValidator:
-    """Validate task data before ingestion."""
-    
-    VALID_PRIORITIES = ["HIGH", "MEDIUM", "LOW"]
-    VALID_TIERS = ["T1_PLANNING", "T2_CODE", "T3_REVIEW", "T4_DEPLOY"]
-    
-    def validate(self, task: Dict) -> Tuple[bool, List[str]]:
-        """Validate a task."""
-        errors = []
-        
-        description = task.get("description", "")
-        if not description:
-            errors.append("Missing required field: description")
-        elif len(description) < 3:
-            errors.append("Description too short (min 3 chars)")
-        elif len(description) > 2000:
-            errors.append("Description too long (max 2000 chars)")
-        
-        priority = task.get("priority")
-        if priority and priority not in self.VALID_PRIORITIES:
-            errors.append(f"Invalid priority: {priority}")
-        
-        tier = task.get("tier")
-        if tier and tier not in self.VALID_TIERS:
-            errors.append(f"Invalid tier: {tier}")
-        
-        return (len(errors) == 0, errors)
-
-
-class DedupEngine:
-    """Deduplication engine with LRU cache."""
-    
-    def __init__(self, cache_size: int = 10000, threshold: float = 0.85):
-        self.cache_size = cache_size
-        self.threshold = threshold
-        self.hash_cache: Dict[str, str] = {}
-        self.access_order: List[str] = []
-        self.lock = threading.Lock()
-    
-    @staticmethod
-    def normalize_for_hash(description: str) -> str:
-        """Normalize description for exact hash comparison."""
-        text = description.lower()
-        text = re.sub(r'[^\w\s]', '', text)
-        text = re.sub(r'\s+', ' ', text).strip()
-        stopwords = {'the', 'a', 'an', 'is', 'are', 'was', 'were', 'to', 'for', 'of', 'in', 'on', 'with'}
-        words = [w for w in text.split() if w not in stopwords]
-        return ' '.join(sorted(words))
-    
-    @staticmethod
-    def compute_dedup_key(description: str) -> str:
-        """Compute SHA256 dedup key."""
-        normalized = DedupEngine.normalize_for_hash(description)
-        return hashlib.sha256(normalized.encode()).hexdigest()[:16]
-    
-    def check_duplicate(
-        self,
-        description: str,
-        existing_tasks: List[Dict],
-    ) -> DuplicateCheckResult:
-        """Check if description is duplicate of existing tasks."""
-        
-        new_hash = self.compute_dedup_key(description)
-        
-        # Level 1: Check cache first
-        with self.lock:
-            cached = self.hash_cache.get(new_hash)
-            if cached:
-                return DuplicateCheckResult(
-                    is_duplicate=True,
-                    matching_task_id=cached,
-                    similarity=1.0,
-                    match_type="exact_cached"
-                )
-        
-        # Level 2: Check against existing tasks
-        for task in existing_tasks:
-            source = task.get("ingestion_source", {})
-            task_hash = source.get("dedup_key")
-            
-            if task_hash == new_hash:
-                self._cache_put(new_hash, task["id"])
-                return DuplicateCheckResult(
-                    is_duplicate=True,
-                    matching_task_id=task["id"],
-                    similarity=1.0,
-                    match_type="exact"
-                )
-        
-        return DuplicateCheckResult(is_duplicate=False)
-    
-    def _cache_put(self, dedup_key: str, task_id: str) -> None:
-        """Add to cache with LRU eviction."""
-        with self.lock:
-            if dedup_key in self.hash_cache:
-                self.access_order.remove(dedup_key)
-            elif len(self.hash_cache) >= self.cache_size:
-                oldest = self.access_order.pop(0)
-                del self.hash_cache[oldest]
-            
-            self.hash_cache[dedup_key] = task_id
-            self.access_order.append(dedup_key)
-    
-    def warm_up(self, tasks: List[Dict]) -> None:
-        """Pre-populate cache from existing tasks."""
-        for task in tasks[-self.cache_size:]:
-            source = task.get("ingestion_source", {})
-            dedup_key = source.get("dedup_key")
-            if dedup_key:
-                self._cache_put(dedup_key, task["id"])
-
-
-class PlanningParser:
-    """Parser for markdown planning documents."""
-    
-    PRIORITY_PATTERNS = {
-        "HIGH": [r'\bHIGH\b', r'\bP0\b', r'\bURGENT\b', r'\bCRITICAL\b'],
-        "LOW": [r'\bLOW\b', r'\bP2\b', r'\bNICE\s*TO\s*HAVE\b'],
-    }
-    
-    TIER_PATTERNS = {
-        "T1_PLANNING": [r'\bT1\b', r'\bPLANNING\b', r'\bDESIGN\b', r'\bARCHITECT\b'],
-        "T2_CODE": [r'\bT2\b', r'\bCODE\b', r'\bIMPLEMENT\b', r'\bBUILD\b'],
-        "T3_REVIEW": [r'\bT3\b', r'\bREVIEW\b', r'\bTEST\b', r'\bVERIFY\b'],
-        "T4_DEPLOY": [r'\bT4\b', r'\bDEPLOY\b', r'\bRELEASE\b', r'\bSHIP\b'],
-    }
-    
-    def parse(
-        self,
-        content: str,
-        file_path: str = None,
-        skip_completed: bool = True,
-    ) -> List[Dict]:
-        """Parse planning document for tasks."""
-        tasks = []
-        lines = content.split('\n')
-        
-        for line_num, line in enumerate(lines, 1):
-            # Match unchecked checkboxes
-            match = re.match(r'^\s*-\s*\[\s*\]\s*(.+)$', line)
-            if match:
-                description = match.group(1).strip()
-                task = self._create_task(
-                    description, line_num, line, file_path
-                )
-                tasks.append(task)
-            
-            # Match checked checkboxes (if not skipping)
-            if not skip_completed:
-                match = re.match(r'^\s*-\s*\[x\]\s*(.+)$', line, re.I)
-                if match:
-                    description = match.group(1).strip()
-                    task = self._create_task(
-                        description, line_num, line, file_path
-                    )
-                    task["status"] = "DONE"
-                    tasks.append(task)
-        
-        return tasks
-    
-    def _create_task(
-        self,
-        description: str,
-        line_num: int,
-        original_line: str,
-        file_path: str,
-    ) -> Dict:
-        """Create task dict from parsed line."""
-        priority = self._infer_priority(description)
-        tier = self._infer_tier(description)
-        
-        return {
-            "description": self._clean_description(description),
-            "priority": priority,
-            "tier": tier,
-            "status": "PENDING",
-            "ingestion_source": {
-                "type": SourceType.PLANNING.value,
-                "file": file_path,
-                "line_number": line_num,
-                "original_text": original_line.strip()[:500],
-            },
-        }
-    
-    def _infer_priority(self, text: str) -> str:
-        """Infer priority from text."""
-        text_upper = text.upper()
-        for priority, patterns in self.PRIORITY_PATTERNS.items():
-            for pattern in patterns:
-                if re.search(pattern, text_upper):
-                    return priority
-        return "MEDIUM"
-    
-    def _infer_tier(self, text: str) -> str:
-        """Infer tier from text."""
-        text_upper = text.upper()
-        for tier, patterns in self.TIER_PATTERNS.items():
-            for pattern in patterns:
-                if re.search(pattern, text_upper):
-                    return tier
-        return "T2_CODE"
-    
-    def _clean_description(self, description: str) -> str:
-        """Remove priority/tier annotations from description."""
-        text = description
-        for patterns in self.PRIORITY_PATTERNS.values():
-            for pattern in patterns:
-                text = re.sub(pattern, '', text, flags=re.I)
-        for patterns in self.TIER_PATTERNS.values():
-            for pattern in patterns:
-                text = re.sub(pattern, '', text, flags=re.I)
-        text = re.sub(r'\s*-\s*$', '', text)
-        text = re.sub(r'\s+', ' ', text).strip()
-        return text
-
-
-class TodoParser:
-    """Parser for TODO/FIXME/HACK comments in code."""
-    
-    PATTERNS = [
-        (r'#\s*(TODO|FIXME|HACK)[\s:]*(.+?)(?:\n|$)', 'python'),
-        (r'//\s*(TODO|FIXME|HACK)[\s:]*(.+?)(?:\n|$)', 'js'),
-        (r'/\*\s*(TODO|FIXME|HACK)[\s:]*(.+?)\*/', 'c'),
-    ]
-    
-    PRIORITY_MAP = {
-        "TODO": "MEDIUM",
-        "FIXME": "HIGH",
-        "HACK": "MEDIUM",
-    }
-    
-    def parse(
-        self,
-        content: str,
-        file_path: str = None,
-    ) -> List[Dict]:
-        """Parse code for TODO comments."""
-        tasks = []
-        
-        for pattern, lang in self.PATTERNS:
-            for match in re.finditer(pattern, content, re.IGNORECASE):
-                tag = match.group(1).upper()
-                description = match.group(2).strip()
-                
-                # Find line number
-                line_num = content[:match.start()].count('\n') + 1
-                
-                task = {
-                    "description": f"[{tag}] {description}",
-                    "priority": self.PRIORITY_MAP.get(tag, "MEDIUM"),
-                    "tier": "T2_CODE",
-                    "status": "PENDING",
-                    "ingestion_source": {
-                        "type": SourceType.TODOS.value,
-                        "file": file_path,
-                        "line_number": line_num,
-                        "original_text": match.group(0).strip()[:500],
-                    },
-                }
-                tasks.append(task)
-        
-        return tasks
-
-
-class HandoffParser:
-    """Parser for agent handoff JSON."""
-    
-    def parse(
-        self,
-        content: str,
-        session_id: str = None,
-    ) -> List[Dict]:
-        """Parse handoff JSON for tasks."""
-        try:
-            data = json.loads(content)
-        except json.JSONDecodeError as e:
-            raise ValueError(f"Invalid handoff JSON: {e}")
-        
-        tasks = []
-        from_session = data.get("from_session", "unknown")
-        to_session = data.get("to_session")
-        context = data.get("context", "")
-        
-        for task_data in data.get("tasks", []):
-            description = task_data.get("description", "")
-            if not description:
-                continue
-            
-            task = {
-                "description": description,
-                "priority": task_data.get("priority", "MEDIUM"),
-                "tier": task_data.get("tier", "T2_CODE"),
-                "status": "PENDING",
-                "context_summary": {
-                    "generated_at": self._now_iso(),
-                    "generated_by": from_session,
-                    "summary": context,
-                    "handoff_notes": task_data.get("notes", ""),
-                },
-                "ingestion_source": {
-                    "type": SourceType.HANDOFFS.value,
-                    "ingested_by": session_id or from_session,
-                    "target_session": to_session,
-                    "original_text": json.dumps(task_data)[:500],
-                },
-            }
-            tasks.append(task)
-        
-        return tasks
-    
-    def _now_iso(self) -> str:
-        return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
-
-
-class MeetingParser:
-    """Parser for meeting notes / action items."""
-    
-    ACTION_PATTERNS = [
-        r'@(\w+)[\s:]+(.+?)(?:\n|$)',  # @person: action
-        r'-\s*\[?\s*\]?\s*@(\w+)[\s:]+(.+?)(?:\n|$)',  # - [ ] @person: action
-        r'ACTION[\s:]+@?(\w+)[\s:]+(.+?)(?:\n|$)',  # ACTION: person: desc
-    ]
-    
-    def parse(
-        self,
-        content: str,
-        file_path: str = None,
-    ) -> List[Dict]:
-        """Parse meeting notes for action items."""
-        tasks = []
-        
-        for pattern in self.ACTION_PATTERNS:
-            for match in re.finditer(pattern, content, re.IGNORECASE):
-                assignee = match.group(1)
-                description = match.group(2).strip()
-                line_num = content[:match.start()].count('\n') + 1
-                
-                task = {
-                    "description": description,
-                    "priority": "MEDIUM",
-                    "tier": "T1_PLANNING",
-                    "status": "PENDING",
-                    "claimed_by": None,  # Don't auto-assign from @mention
-                    "ingestion_source": {
-                        "type": SourceType.MEETINGS.value,
-                        "file": file_path,
-                        "line_number": line_num,
-                        "original_text": match.group(0).strip()[:500],
-                        "mentioned_assignee": assignee,
-                    },
-                }
-                tasks.append(task)
-        
-        return tasks
-
-
-class ApiParser:
-    """Parser for external API payloads (Jira, Linear, GitHub)."""
-    
-    def parse(
-        self,
-        payload: Dict,
-        source_name: str = "api",
-    ) -> List[Dict]:
-        """Parse API payload for tasks."""
-        tasks = []
-        
-        # Handle different API formats
-        items = payload.get("issues") or payload.get("tickets") or payload.get("items") or []
-        
-        for item in items:
-            # Normalize field names
-            description = (
-                item.get("summary") or 
-                item.get("title") or 
-                item.get("description") or 
-                item.get("name") or ""
-            )
-            
-            if not description:
-                continue
-            
-            priority = self._normalize_priority(
-                item.get("priority") or item.get("severity") or "Medium"
-            )
-            
-            external_id = (
-                item.get("key") or 
-                item.get("id") or 
-                item.get("number") or ""
-            )
-            
-            task = {
-                "description": description,
-                "priority": priority,
-                "tier": "T2_CODE",
-                "status": "PENDING",
-                "ingestion_source": {
-                    "type": SourceType.API.value,
-                    "external_id": str(external_id),
-                    "api_source": source_name,
-                    "original_text": json.dumps(item)[:500],
-                },
-            }
-            tasks.append(task)
-        
-        return tasks
-    
-    def _normalize_priority(self, priority: str) -> str:
-        """Normalize priority from various API formats."""
-        priority_lower = str(priority).lower()
-        if priority_lower in ["highest", "critical", "blocker", "p0", "urgent"]:
-            return "HIGH"
-        if priority_lower in ["lowest", "trivial", "minor", "p2", "low"]:
-            return "LOW"
-        return "MEDIUM"
-
-
-class IngestionBatch:
-    """Track operations in an ingestion batch."""
-    
-    def __init__(self, batch_id: str, source_type: str, session_id: str):
-        self.batch_id = batch_id
-        self.source_type = source_type
-        self.session_id = session_id
-        self.created_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
-        self.created_task_ids: List[str] = []
-        self.updated_tasks: List[Dict] = []
-        self.skipped_task_ids: List[str] = []
-        self.failed_tasks: List[Dict] = []
-        self.source_file: str = None
-        self.completed_at: str = None
-        self.rolled_back_at: str = None
-    
-    def record_create(self, task_id: str) -> None:
-        self.created_task_ids.append(task_id)
-    
-    def record_update(self, task_id: str, field: str, old_val: Any, new_val: Any) -> None:
-        self.updated_tasks.append({
-            "task_id": task_id, "field": field,
-            "old_value": old_val, "new_value": new_val,
-        })
-    
-    def record_skip(self, task_id: str) -> None:
-        self.skipped_task_ids.append(task_id)
-    
-    def record_failure(self, description: str, error: str) -> None:
-        self.failed_tasks.append({"description": description[:200], "error": error})
-    
-    def to_dict(self) -> Dict:
-        return {
-            "batch_id": self.batch_id,
-            "source_type": self.source_type,
-            "session_id": self.session_id,
-            "source_file": self.source_file,
-            "created_at": self.created_at,
-            "completed_at": self.completed_at,
-            "rolled_back_at": self.rolled_back_at,
-            "stats": {
-                "created": len(self.created_task_ids),
-                "updated": len(self.updated_tasks),
-                "skipped": len(self.skipped_task_ids),
-                "failed": len(self.failed_tasks),
-            },
-            "created_task_ids": self.created_task_ids,
-        }
-
-
-class TaskIngestionEngine:
-    """
-    Enterprise-grade multi-source task ingestion engine.
-    
-    Scales: 1 → 1000 sources, 10K tasks/sec
-    """
-    
-    def __init__(
-        self,
-        task_store: Any = None,
-        agent_pool: Any = None,
-        brain_path: Path = None,
-        enable_embeddings: bool = False,
-        dedup_threshold: float = 0.85,
-    ):
-        self.task_store = task_store
-        self.agent_pool = agent_pool
-        self.brain_path = brain_path
-        self.enable_embeddings = enable_embeddings
-        
-        self.sanitizer = InputSanitizer()
-        self.validator = TaskValidator()
-        self.dedup_engine = DedupEngine(threshold=dedup_threshold)
-        
-        self.parsers = {
-            SourceType.PLANNING.value: PlanningParser(),
-            SourceType.TODOS.value: TodoParser(),
-            SourceType.HANDOFFS.value: HandoffParser(),
-            SourceType.MEETINGS.value: MeetingParser(),
-            SourceType.API.value: ApiParser(),
-        }
-        
-        self.batches: Dict[str, IngestionBatch] = {}
-        self.lock = threading.RLock()
-        
-        self.stats = {
-            "total_ingested": 0,
-            "total_skipped": 0,
-            "total_failed": 0,
-            "by_source": defaultdict(int),
-        }
-    
-    def ingest_from_file(
-        self,
-        file_path: str,
-        source_type: str = "auto",
-        session_id: str = None,
-        auto_assign: bool = False,
-        skip_dedup: bool = False,
-        dry_run: bool = False,
-    ) -> IngestionResult:
-        """Ingest tasks from a file."""
-        file_path = self.sanitizer.sanitize_file_path(file_path)
-        
-        if not os.path.exists(file_path):
-            return IngestionResult(
-                success=False,
-                batch_id="",
-                errors=[f"File not found: {file_path}"]
-            )
-        
-        with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
-            content = f.read()
-        
-        if source_type == "auto":
-            source_type = self._detect_source_type(file_path, content)
-        
-        return self.ingest_from_text(
-            text=content,
-            source_type=source_type,
-            session_id=session_id,
-            metadata={"file": file_path},
-            auto_assign=auto_assign,
-            skip_dedup=skip_dedup,
-            dry_run=dry_run,
-        )
-    
-    def ingest_from_text(
-        self,
-        text: str,
-        source_type: str,
-        session_id: str = None,
-        metadata: Dict = None,
-        auto_assign: bool = False,
-        skip_dedup: bool = False,
-        dry_run: bool = False,
-    ) -> IngestionResult:
-        """Ingest tasks from raw text."""
-        batch_id = self._generate_batch_id()
-        batch = IngestionBatch(batch_id, source_type, session_id or "unknown")
-        
-        if metadata and metadata.get("file"):
-            batch.source_file = metadata["file"]
-        
-        try:
-            # Parse content
-            parser = self.parsers.get(source_type)
-            if not parser:
-                return IngestionResult(
-                    success=False,
-                    batch_id=batch_id,
-                    errors=[f"Unknown source type: {source_type}"]
-                )
-            
-            if source_type == SourceType.HANDOFFS.value:
-                tasks = parser.parse(text, session_id)
-            elif metadata and metadata.get("file"):
-                tasks = parser.parse(text, file_path=metadata["file"])
-            else:
-                tasks = parser.parse(text)
-            
-            return self._process_tasks(
-                tasks, batch, auto_assign, skip_dedup, dry_run
-            )
-            
-        except Exception as e:
-            return IngestionResult(
-                success=False,
-                batch_id=batch_id,
-                errors=[f"Parse error: {str(e)}"]
-            )
-    
-    def ingest_from_api(
-        self,
-        source_name: str,
-        payload: Dict,
-        session_id: str = None,
-        auto_assign: bool = False,
-    ) -> IngestionResult:
-        """Ingest tasks from external API payload."""
-        batch_id = self._generate_batch_id()
-        batch = IngestionBatch(batch_id, SourceType.API.value, session_id or "unknown")
-        
-        try:
-            parser = self.parsers[SourceType.API.value]
-            tasks = parser.parse(payload, source_name)
-            
-            return self._process_tasks(tasks, batch, auto_assign)
-            
-        except Exception as e:
-            return IngestionResult(
-                success=False,
-                batch_id=batch_id,
-                errors=[f"API parse error: {str(e)}"]
-            )
-    
-    def ingest_batch(
-        self,
-        tasks: List[Dict],
-        source_type: str = "manual",
-        session_id: str = None,
-        skip_dedup: bool = False,
-        dry_run: bool = False,
-    ) -> IngestionResult:
-        """Ingest a batch of pre-parsed tasks."""
-        batch_id = self._generate_batch_id()
-        batch = IngestionBatch(batch_id, source_type, session_id or "unknown")
-        
-        return self._process_tasks(tasks, batch, False, skip_dedup, dry_run)
-    
-    def _process_tasks(
-        self,
-        tasks: List[Dict],
-        batch: IngestionBatch,
-        auto_assign: bool = False,
-        skip_dedup: bool = False,
-        dry_run: bool = False,
-    ) -> IngestionResult:
-        """Process parsed tasks through validation, dedup, and storage."""
-        result = IngestionResult(success=True, batch_id=batch.batch_id)
-        
-        # Track seen descriptions within this batch for intra-batch dedup
-        seen_in_batch: Dict[str, str] = {}  # dedup_key -> task_id
-        
-        with self.lock:
-            existing_tasks = self._get_existing_tasks()
-            
-            for task in tasks:
-                try:
-                    # Sanitize
-                    task["description"] = self.sanitizer.sanitize_description(
-                        task.get("description", "")
-                    )
-                    
-                    # Validate
-                    is_valid, errors = self.validator.validate(task)
-                    if not is_valid:
-                        batch.record_failure(task.get("description", "")[:100], "; ".join(errors))
-                        result.tasks_failed += 1
-                        result.errors.extend(errors)
-                        continue
-                    
-                    # Dedup check
-                    if not skip_dedup:
-                        # First check intra-batch duplicates
-                        dedup_key = DedupEngine.compute_dedup_key(task["description"])
-                        if dedup_key in seen_in_batch:
-                            batch.record_skip(seen_in_batch[dedup_key])
-                            result.tasks_skipped += 1
-                            continue
-                        
-                        # Then check against existing tasks
-                        dup_result = self.dedup_engine.check_duplicate(
-                            task["description"], existing_tasks
-                        )
-                        if dup_result.is_duplicate:
-                            batch.record_skip(dup_result.matching_task_id)
-                            result.tasks_skipped += 1
-                            continue
-                    
-                    if dry_run:
-                        result.tasks_created += 1
-                        if not skip_dedup:
-                            seen_in_batch[dedup_key] = f"dry_run_{len(seen_in_batch)}"
-                        continue
-                    
-                    # Create task
-                    task_id = self._create_task(task, batch)
-                    
-                    # Track for intra-batch dedup
-                    if not skip_dedup:
-                        seen_in_batch[dedup_key] = task_id
-                    batch.record_create(task_id)
-                    result.tasks_created += 1
-                    result.created_task_ids.append(task_id)
-                    
-                    # Auto-assign if enabled
-                    if auto_assign and self.agent_pool:
-                        assign_result = self._auto_assign(task_id, task)
-                        if assign_result.get("assigned"):
-                            result.auto_assigned.append({
-                                "task_id": task_id,
-                                "agent_id": assign_result.get("agent_id"),
-                            })
-                    
-                except Exception as e:
-                    batch.record_failure(task.get("description", "")[:100], str(e))
-                    result.tasks_failed += 1
-                    result.errors.append(str(e))
-            
-            # Finalize batch
-            batch.completed_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
-            self.batches[batch.batch_id] = batch
-            result.rollback_id = batch.batch_id
-            
-            # Update stats
-            self.stats["total_ingested"] += result.tasks_created
-            self.stats["total_skipped"] += result.tasks_skipped
-            self.stats["total_failed"] += result.tasks_failed
-            self.stats["by_source"][batch.source_type] += result.tasks_created
-        
-        result.success = result.tasks_failed == 0
-        return result
-    
-    def _create_task(self, task: Dict, batch: IngestionBatch) -> str:
-        """Create task in store."""
-        task_id = f"task-{uuid.uuid4().hex[:8]}"
-        now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
-        
-        # Ensure ingestion_source
-        if "ingestion_source" not in task:
-            task["ingestion_source"] = {}
-        
-        source = task["ingestion_source"]
-        source["type"] = source.get("type", batch.source_type)
-        source["ingested_at"] = now
-        source["ingested_by"] = batch.session_id
-        source["dedup_key"] = DedupEngine.compute_dedup_key(task["description"])
-        source["batch_id"] = batch.batch_id
-        
-        # Full task structure
-        full_task = {
-            "id": task_id,
-            "description": task["description"],
-            "status": task.get("status", "PENDING"),
-            "priority": task.get("priority", "MEDIUM"),
-            "tier": task.get("tier", "T2_CODE"),
-            "blocked_by": task.get("blocked_by", []),
-            "required_skills": task.get("required_skills", []),
-            "claimed_by": task.get("claimed_by"),
-            "source": batch.source_type,
-            "created_at": now,
-            "updated_at": now,
-            "ingestion_source": source,
-            "context_summary": task.get("context_summary"),
-            "checkpoint": None,
-            "dependency_metadata": {"depth": 0, "blocks": []},
-        }
-        
-        # Store task
-        if self.task_store:
-            self.task_store.put(task_id, full_task)
-        
-        # Add to dedup cache
-        self.dedup_engine._cache_put(source["dedup_key"], task_id)
-        
-        return task_id
-    
-    def _auto_assign(self, task_id: str, task: Dict) -> Dict:
-        """Auto-assign task to available agent."""
-        if not self.agent_pool:
-            return {"assigned": False, "reason": "no_pool"}
-        
-        tier = task.get("tier", "T2_CODE")
-        return self.agent_pool.assign_task(task_id, tier=tier)
-    
-    def check_duplicate(self, description: str) -> DuplicateCheckResult:
-        """Check if description would be duplicate."""
-        existing = self._get_existing_tasks()
-        return self.dedup_engine.check_duplicate(description, existing)
-    
-    def rollback(self, batch_id: str, reason: str = None) -> Dict:
-        """Rollback an ingestion batch."""
-        with self.lock:
-            batch = self.batches.get(batch_id)
-            if not batch:
-                return {"success": False, "error": f"Batch {batch_id} not found"}
-            
-            if batch.rolled_back_at:
-                return {"success": False, "error": "Batch already rolled back"}
-            
-            removed = 0
-            for task_id in batch.created_task_ids:
-                if self.task_store:
-                    self.task_store.delete(task_id)
-                removed += 1
-            
-            batch.rolled_back_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
-            
-            return {
-                "success": True,
-                "batch_id": batch_id,
-                "tasks_removed": removed,
-                "reason": reason,
-            }
-    
-    def get_ingestion_stats(self) -> Dict:
-        """Get overall ingestion statistics."""
-        return {
-            "total_ingested": self.stats["total_ingested"],
-            "total_skipped": self.stats["total_skipped"],
-            "total_failed": self.stats["total_failed"],
-            "by_source": dict(self.stats["by_source"]),
-            "batches_count": len(self.batches),
-            "dedup_cache_size": len(self.dedup_engine.hash_cache),
-        }
-    
-    def list_batches(self, limit: int = 10) -> List[Dict]:
-        """List recent ingestion batches."""
-        batches = sorted(
-            self.batches.values(),
-            key=lambda b: b.created_at,
-            reverse=True
-        )[:limit]
-        return [b.to_dict() for b in batches]
-    
-    def _detect_source_type(self, file_path: str, content: str) -> str:
-        """Auto-detect source type from file/content."""
-        path = Path(file_path)
-        filename = path.name.lower()
-        ext = path.suffix.lower()
-        
-        if "handoff" in filename:
-            return SourceType.HANDOFFS.value
-        if "meeting" in filename or "standup" in filename:
-            return SourceType.MEETINGS.value
-        if "planning" in filename or "sprint" in filename or "roadmap" in filename:
-            return SourceType.PLANNING.value
-        
-        if ext in [".py", ".js", ".ts", ".go", ".rs", ".java", ".c", ".cpp"]:
-            return SourceType.TODOS.value
-        if ext == ".json":
-            return SourceType.API.value
-        if ext == ".md":
-            if "- [ ]" in content or "- [x]" in content:
-                return SourceType.PLANNING.value
-            if "@" in content and ("action" in content.lower() or "item" in content.lower()):
-                return SourceType.MEETINGS.value
-            return SourceType.PLANNING.value
-        
-        return SourceType.MANUAL.value
-    
-    def _generate_batch_id(self) -> str:
-        """Generate unique batch ID."""
-        timestamp = time.strftime("%Y%m%d_%H%M%S")
-        suffix = uuid.uuid4().hex[:6]
-        return f"batch_{timestamp}_{suffix}"
-    
-    def _get_existing_tasks(self) -> List[Dict]:
-        """Get existing tasks for dedup check."""
-        if self.task_store:
-            return list(self.task_store.get_all().values())
-        return []
-
-
-def format_ingestion_result(result: IngestionResult) -> str:
-    """Format ingestion result for MCP tool output."""
-    lines = []
-    
-    if result.success:
-        lines.append(f"✅ Ingestion Complete - Batch: {result.batch_id}")
-    else:
-        lines.append(f"⚠️ Ingestion Partial - Batch: {result.batch_id}")
-    
-    lines.append("=" * 50)
-    lines.append("\n📊 **Results:**")
-    lines.append(f"   Created: {result.tasks_created}")
-    lines.append(f"   Updated: {result.tasks_updated}")
-    lines.append(f"   Skipped: {result.tasks_skipped} (duplicates)")
-    
-    if result.tasks_failed > 0:
-        lines.append(f"   ❌ Failed: {result.tasks_failed}")
-    
-    if result.errors:
-        lines.append("\n⚠️ **Errors:**")
-        for error in result.errors[:5]:
-            lines.append(f"   - {error}")
-        if len(result.errors) > 5:
-            lines.append(f"   ... and {len(result.errors) - 5} more")
-    
-    if result.auto_assigned:
-        lines.append("\n🤖 **Auto-Assigned:**")
-        for assignment in result.auto_assigned[:10]:
-            lines.append(f"   - {assignment['task_id']} → {assignment['agent_id']}")
-    
-    lines.append(f"\n💡 To undo: brain_rollback_ingestion('{result.batch_id}')")
-    
-    return "\n".join(lines)