@mc1global/opencode-jarvis 0.13.0 → 0.13.1

package/scripts/rag/domain_mapper/persistence.py

@@ -0,0 +1,167 @@
+"""Persistence for Domain Mapper.
+
+Handles saving domain maps to files and database.
+"""
+
+import json
+import logging
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any, Dict
+
+from .models import DomainMapTOON
+
+logger = logging.getLogger(__name__)
+
+
+def save_domain_map_atomic(
+    domain_map: DomainMapTOON,
+    output_path: str,
+    verbose: bool = False
+) -> None:
+    """
+    Save domain map to file with atomic write.
+
+    Uses temp file + os.replace for crash-safe writing.
+
+    Args:
+        domain_map: Pydantic DomainMapTOON model
+        output_path: Target file path
+        verbose: Enable verbose logging
+
+    Raises:
+        IOError: If write operation fails
+    """
+    json_output = domain_map.model_dump_json(indent=2)
+
+    # Atomic write
+    output_dir = os.path.dirname(output_path) or "."
+    temp_fd, temp_path = tempfile.mkstemp(
+        dir=output_dir,
+        prefix='.tmp_domain_map_',
+        suffix='.json'
+    )
+
+    try:
+        with os.fdopen(temp_fd, 'w') as f:
+            f.write(json_output)
+
+        os.replace(temp_path, output_path)
+        if verbose:
+            logger.info(f"Domain map saved to {output_path}")
+        else:
+            print(f"ok:saved:{output_path}")
+
+    except Exception as e:
+        if os.path.exists(temp_path):
+            os.unlink(temp_path)
+        if verbose:
+            raise IOError(f"Failed to save domain map: {e}")
+        else:
+            print(f"error:save_failed:{str(e)[:50]}")
+            sys.exit(1)
+
+
+def save_workspace_map(
+    workspace_map: Dict[str, Any],
+    output_path: str,
+    verbose: bool = False
+) -> None:
+    """
+    Save workspace map to file with atomic write.
+
+    Args:
+        workspace_map: Dictionary with workspace structure
+        output_path: Target file path
+        verbose: Enable verbose logging
+    """
+    json_output = json.dumps(workspace_map, indent=2)
+
+    output_dir = os.path.dirname(output_path) or "."
+    os.makedirs(output_dir, exist_ok=True)
+
+    temp_fd, temp_path = tempfile.mkstemp(
+        dir=output_dir,
+        prefix='.tmp_workspace_map_',
+        suffix='.json'
+    )
+
+    try:
+        with os.fdopen(temp_fd, 'w') as f:
+            f.write(json_output)
+
+        os.replace(temp_path, output_path)
+        if verbose:
+            logger.info(f"Workspace map saved to {output_path}")
+        else:
+            print(f"ok:saved:{output_path}")
+
+    except Exception as e:
+        if os.path.exists(temp_path):
+            os.unlink(temp_path)
+        if verbose:
+            raise IOError(f"Failed to save workspace map: {e}")
+        else:
+            print(f"error:save_failed:{str(e)[:50]}")
+            sys.exit(1)
+
+
+def persist_to_database(
+    workspace_name: str,
+    verbose: bool = False
+) -> None:
+    """
+    Persist domain maps from JSON to database.
+
+    Calls domain_map_db.py rebuild-from-rag to load JSON into DB.
+
+    Args:
+        workspace_name: Workspace identifier
+        verbose: Enable verbose logging
+
+    Raises:
+        ValueError: If DB write fails
+    """
+    try:
+        project_root = Path(__file__).resolve().parent.parent.parent.parent
+        cmd = [
+            sys.executable,
+            str(project_root / "scripts" / "managers" / "domain_map_db.py"),
+            "rebuild-from-rag",
+            "-w",
+            workspace_name
+        ]
+
+        if verbose:
+            logger.info(f"Persisting to database: {' '.join(cmd)}")
+
+        result = subprocess.run(
+            cmd,
+            cwd=project_root,
+            capture_output=True,
+            text=True,
+            check=True
+        )
+
+        if verbose:
+            logger.info(f"Database persist output: {result.stdout}")
+        else:
+            print(f"ok:db_persist:{result.stdout.strip()}")
+
+    except subprocess.CalledProcessError as e:
+        error_msg = f"Failed to persist to database: {e.stderr}"
+        if verbose:
+            logger.error(error_msg)
+        else:
+            print(f"error:db_persist_failed:{e.stderr[:100]}")
+        raise ValueError(error_msg)
+    except Exception as e:
+        error_msg = f"Database persistence error: {str(e)}"
+        if verbose:
+            logger.error(error_msg)
+        else:
+            print(f"error:db_persist_error:{str(e)[:100]}")
+        raise

package/scripts/rag/domain_mapper/vectorstore.py

@@ -0,0 +1,82 @@
+"""Vector Store Loading for Domain Mapper.
+
+Handles ChromaDB vector store initialization for semantic search.
+"""
+
+import logging
+import os
+import sys
+from typing import Optional
+
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_chroma import Chroma
+
+from .config import DomainMapperConfig, load_config
+
+# Import PathRegistry for workspace-based path resolution
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+from src.shared.paths import PathRegistry
+
+logger = logging.getLogger(__name__)
+
+
+def load_vectorstore(
+    config: DomainMapperConfig = None,
+    verbose: bool = False,
+    workspace: Optional[str] = None
+) -> Chroma:
+    """
+    Load existing Chroma vector store.
+
+    Args:
+        config: Domain mapper configuration
+        verbose: Enable verbose logging
+        workspace: Workspace name for per-workspace RAG isolation.
+                  If provided, uses workspace-specific ChromaDB path.
+                  If None, falls back to config.vectorstore.directory (legacy).
+
+    Returns:
+        Initialized Chroma instance
+
+    Raises:
+        SystemExit: If vector store not found or unavailable
+    """
+    if config is None:
+        config = load_config(verbose=verbose)
+
+    # Determine store directory: workspace-based or legacy
+    if workspace:
+        store_dir = str(PathRegistry.get_workspace_rag_path(workspace))
+        if verbose:
+            logger.info(f"Using per-workspace RAG path for '{workspace}': {store_dir}")
+    else:
+        store_dir = config.vectorstore.directory
+
+    if not os.path.exists(store_dir):
+        if verbose:
+            logger.error(f"Vector store not found: {store_dir}")
+            logger.error("Run rag_cli.py index first to index code")
+        else:
+            print(f"error:vectorstore_not_found:{store_dir}")
+        sys.exit(1)
+
+    try:
+        embeddings = OllamaEmbeddings(
+            model=config.ollama.embedding_model,
+            base_url=config.ollama.host_url
+        )
+        vectorstore = Chroma(
+            persist_directory=store_dir,
+            collection_name="jarvis_rag",
+            embedding_function=embeddings
+        )
+        if verbose:
+            logger.info(f"Vector store loaded from {store_dir}")
+        return vectorstore
+    except Exception as e:
+        if verbose:
+            logger.error(f"Failed to load vector store: {e}")
+            logger.error("Ensure Ollama is running")
+        else:
+            print(f"error:vectorstore_load_failed:{str(e)[:50]}")
+        sys.exit(1)

package/scripts/rag/index_external_sources.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+"""
+External Sources Indexer for RAG
+
+Batch indexes external projects defined in config/rag_external_sources.yaml.
+
+Usage:
+    python scripts/rag/index_external_sources.py                    # Index all enabled
+    python scripts/rag/index_external_sources.py --source=kotlin_lib # Index specific
+    python scripts/rag/index_external_sources.py --group=kotlin_ecosystem # Index group
+    python scripts/rag/index_external_sources.py --list             # List sources
+    python scripts/rag/index_external_sources.py --status           # Show status
+"""
+
+import sys
+import argparse
+import subprocess
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import yaml
+
+# Add project root to path
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+
+class ExternalSourcesIndexer:
+    """Indexes external sources defined in configuration."""
+
+    def __init__(self):
+        self.config_path = project_root / "config" / "rag_external_sources.yaml"
+        self.config = self._load_config()
+
+    def _load_config(self) -> Dict:
+        """Load external sources configuration."""
+        if not self.config_path.exists():
+            return {"external_sources": {}, "groups": {}, "defaults": {}}
+
+        with open(self.config_path) as f:
+            return yaml.safe_load(f) or {}
+
+    def _expand_path(self, path: str) -> Path:
+        """Expand ~ and resolve path."""
+        return Path(path).expanduser().resolve()
+
+    def _get_source(self, name: str) -> Optional[Dict]:
+        """Get source configuration by name."""
+        sources = self.config.get("external_sources", {})
+        return sources.get(name)
+
+    def _get_enabled_sources(self) -> Dict[str, Dict]:
+        """Get all enabled sources."""
+        sources = self.config.get("external_sources", {})
+        defaults = self.config.get("defaults", {})
+
+        enabled = {}
+        for name, source in sources.items():
+            if source is None:
+                continue
+            is_enabled = source.get("enabled", defaults.get("enabled", True))
+            if is_enabled:
+                enabled[name] = source
+
+        return enabled
+
+    def _get_group_sources(self, group_name: str) -> List[str]:
+        """Get source names in a group."""
+        groups = self.config.get("groups", {})
+        group = groups.get(group_name, {})
+        return group.get("sources", [])
+
+    def _index_source(self, name: str, source: Dict) -> bool:
+        """Index a single source."""
+        path = self._expand_path(source.get("path", ""))
+        workspace = source.get("workspace", name)
+
+        if not path.exists():
+            print(f"error:source:{name}:path_not_found:{path}")
+            return False
+
+        # Build command
+        cmd = [
+            sys.executable,
+            str(project_root / "scripts" / "rag" / "rag_cli.py"),
+            "index",
+            "-d", str(path),
+            "-w", workspace
+        ]
+
+        # Run indexing
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                cwd=str(project_root)
+            )
+
+            if result.returncode == 0:
+                print(f"ok:indexed:{name}:workspace:{workspace}")
+                return True
+            else:
+                error_msg = result.stderr.strip() or result.stdout.strip()
+                print(f"error:source:{name}:indexing_failed:{error_msg[:100]}")
+                return False
+
+        except Exception as e:
+            print(f"error:source:{name}:exception:{str(e)}")
+            return False
+
+    def index_all(self) -> str:
+        """Index all enabled sources."""
+        sources = self._get_enabled_sources()
+
+        if not sources:
+            return "ok:no_sources_configured"
+
+        success = 0
+        failed = 0
+
+        for name, source in sources.items():
+            if self._index_source(name, source):
+                success += 1
+            else:
+                failed += 1
+
+        return f"ok:indexed:{success}:failed:{failed}"
+
+    def index_source(self, name: str) -> str:
+        """Index a specific source by name."""
+        source = self._get_source(name)
+
+        if not source:
+            return f"error:source_not_found:{name}"
+
+        if self._index_source(name, source):
+            return f"ok:indexed:{name}"
+        else:
+            return f"error:indexing_failed:{name}"
+
+    def index_group(self, group_name: str) -> str:
+        """Index all sources in a group."""
+        source_names = self._get_group_sources(group_name)
+
+        if not source_names:
+            return f"error:group_not_found_or_empty:{group_name}"
+
+        success = 0
+        failed = 0
+
+        for name in source_names:
+            source = self._get_source(name)
+            if source and self._index_source(name, source):
+                success += 1
+            else:
+                failed += 1
+
+        return f"ok:group:{group_name}:indexed:{success}:failed:{failed}"
+
+    def list_sources(self) -> str:
+        """List all configured sources."""
+        sources = self.config.get("external_sources", {})
+
+        if not sources:
+            return "ok:no_sources_configured"
+
+        names = [name for name in sources.keys() if sources[name] is not None]
+        return f"ok:count:{len(names)}:sources:[{','.join(names)}]"
+
+    def list_groups(self) -> str:
+        """List all configured groups."""
+        groups = self.config.get("groups", {})
+
+        if not groups:
+            return "ok:no_groups_configured"
+
+        names = list(groups.keys())
+        return f"ok:count:{len(names)}:groups:[{','.join(names)}]"
+
+    def get_status(self) -> str:
+        """Get status of all sources."""
+        sources = self.config.get("external_sources", {})
+        defaults = self.config.get("defaults", {})
+
+        if not sources:
+            return "ok:no_sources_configured"
+
+        enabled_count = 0
+        disabled_count = 0
+        missing_count = 0
+
+        for name, source in sources.items():
+            if source is None:
+                continue
+
+            is_enabled = source.get("enabled", defaults.get("enabled", True))
+            path = self._expand_path(source.get("path", ""))
+
+            if not path.exists():
+                missing_count += 1
+            elif is_enabled:
+                enabled_count += 1
+            else:
+                disabled_count += 1
+
+        return f"ok:enabled:{enabled_count}:disabled:{disabled_count}:missing:{missing_count}"
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Index external sources for RAG",
+        formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+
+    parser.add_argument(
+        "--source",
+        help="Index specific source by name"
+    )
+    parser.add_argument(
+        "--group",
+        help="Index all sources in a group"
+    )
+    parser.add_argument(
+        "--list",
+        action="store_true",
+        help="List all configured sources"
+    )
+    parser.add_argument(
+        "--list-groups",
+        action="store_true",
+        help="List all configured groups"
+    )
+    parser.add_argument(
+        "--status",
+        action="store_true",
+        help="Show status of all sources"
+    )
+    parser.add_argument(
+        "--all",
+        action="store_true",
+        help="Index all enabled sources"
+    )
+
+    args = parser.parse_args()
+    indexer = ExternalSourcesIndexer()
+
+    if args.list:
+        print(indexer.list_sources())
+    elif args.list_groups:
+        print(indexer.list_groups())
+    elif args.status:
+        print(indexer.get_status())
+    elif args.source:
+        print(indexer.index_source(args.source))
+    elif args.group:
+        print(indexer.index_group(args.group))
+    elif args.all:
+        print(indexer.index_all())
+    else:
+        # Default: index all enabled
+        print(indexer.index_all())
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/rag/index_menu_ocr_hybrid.sh

@@ -0,0 +1,191 @@
+#!/bin/bash
+
+################################################################################
+# Menu OCR - Indexação Híbrida Interativa
+################################################################################
+
+set -e
+
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+RED='\033[0;31m'
+BOLD='\033[1m'
+NC='\033[0m'
+
+print_header() {
+    echo ""
+    printf "${CYAN}========================================${NC}\n"
+    printf "${CYAN}%s${NC}\n" "$1"
+    printf "${CYAN}========================================${NC}\n"
+}
+
+print_workspace() {
+    echo ""
+    printf "${BLUE}>>> WORKSPACE: %s${NC}\n" "$1"
+}
+
+index_workspace() {
+    local workspace=$1
+    
+    print_workspace "$workspace"
+    
+    local start=$(date +%s)
+    
+    if python scripts/rag/rag_cli.py index -w "$workspace" --force 2>&1 | grep -q "ok:"; then
+        local end=$(date +%s)
+        local duration=$((end - start))
+        printf "${GREEN}  ✓ Indexed in ${duration}s${NC}\n"
+        return 0
+    else
+        printf "${RED}  ✗ Failed${NC}\n"
+        return 1
+    fi
+}
+
+show_status() {
+    local workspace=$1
+    printf "${CYAN}Status for %s:${NC}\n" "$workspace"
+    python scripts/rag/rag_cli.py status -w "$workspace" 2>&1 | grep "ok:rag:status" || printf "${YELLOW}  Not indexed yet${NC}\n"
+}
+
+################################################################################
+# Main Menu
+################################################################################
+
+while true; do
+    clear
+    print_header "MENU OCR - HYBRID INDEXING (Interactive)"
+    
+    echo ""
+    printf "${BOLD}Available Workspaces:${NC}\n"
+    echo ""
+    printf "  ${BOLD}Specific Modules:${NC}\n"
+    echo "    1) menu-ocr-backend      - Backend (TypeScript + Express)"
+    echo "    2) menu-ocr-admin        - Admin Portal (Vue 3)"
+    echo "    3) menu-ocr-client       - Client Portal (Vue 3)"
+    echo "    4) menu-ocr-frontend     - Frontend Alternative (Vue 3 + DDD)"
+    echo "    5) menu-ocr-python       - Python Client SDK"
+    echo "    6) menu-ocr-docs         - Documentation"
+    echo "    7) menu-ocr-deployment   - Deployment Scripts (GCP)"
+    echo ""
+    printf "  ${BOLD}Aggregator:${NC}\n"
+    echo "    8) menu-ocr              - ALL modules (aggregator)"
+    echo ""
+    printf "  ${BOLD}Batch Operations:${NC}\n"
+    echo "    9) Index ALL (modules + aggregator)"
+    echo ""
+    printf "  ${BOLD}Utilities:${NC}\n"
+    echo "    s) Show status of all workspaces"
+    echo "    c) Clean all RAG databases"
+    echo "    q) Quit"
+    echo ""
+    printf "${YELLOW}Select option: ${NC}"
+    
+    read -r option
+    
+    case $option in
+        1)
+            index_workspace "menu-ocr-backend"
+            ;;
+        2)
+            index_workspace "menu-ocr-admin"
+            ;;
+        3)
+            index_workspace "menu-ocr-client"
+            ;;
+        4)
+            index_workspace "menu-ocr-frontend"
+            ;;
+        5)
+            index_workspace "menu-ocr-python"
+            ;;
+        6)
+            index_workspace "menu-ocr-docs"
+            ;;
+        7)
+            index_workspace "menu-ocr-deployment"
+            ;;
+        8)
+            index_workspace "menu-ocr"
+            ;;
+        9)
+            print_header "Indexing ALL Workspaces"
+            echo ""
+            echo "This will index 8 workspaces (may take 10-15 minutes)"
+            echo ""
+            read -p "Continue? (y/N): " confirm
+            
+            if [[ $confirm == [yY] ]]; then
+                TOTAL_START=$(date +%s)
+                
+                echo ""
+                printf "${CYAN}Cleaning existing databases...${NC}\n"
+                rm -rf workspace/menu-ocr*/chroma_db*
+                printf "${GREEN}✓ Cleaned${NC}\n"
+                
+                echo ""
+                printf "${CYAN}Indexing module-specific workspaces...${NC}\n"
+                
+                index_workspace "menu-ocr-backend"
+                index_workspace "menu-ocr-admin"
+                index_workspace "menu-ocr-client"
+                index_workspace "menu-ocr-frontend"
+                index_workspace "menu-ocr-python"
+                index_workspace "menu-ocr-docs"
+                index_workspace "menu-ocr-deployment"
+                
+                echo ""
+                printf "${CYAN}Indexing aggregator workspace...${NC}\n"
+                index_workspace "menu-ocr"
+                
+                TOTAL_END=$(date +%s)
+                TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
+                
+                echo ""
+                printf "${GREEN}✓ ALL WORKSPACES INDEXED${NC}\n"
+                printf "${CYAN}Total time: ${TOTAL_DURATION}s${NC}\n"
+            fi
+            ;;
+        s|S)
+            print_header "RAG Status Summary"
+            echo ""
+            
+            for ws in menu-ocr-backend menu-ocr-admin menu-ocr-client menu-ocr-frontend menu-ocr-python menu-ocr-docs menu-ocr-deployment menu-ocr; do
+                show_status "$ws"
+                echo ""
+            done
+            ;;
+        c|C)
+            print_header "Clean RAG Databases"
+            echo ""
+            printf "${RED}This will DELETE all indexed data!${NC}\n"
+            echo ""
+            read -p "Are you sure? (y/N): " confirm
+            
+            if [[ $confirm == [yY] ]]; then
+                rm -rf workspace/menu-ocr*/chroma_db*
+                echo ""
+                printf "${GREEN}✓ All RAG databases cleaned${NC}\n"
+            else
+                echo ""
+                printf "${YELLOW}Cancelled${NC}\n"
+            fi
+            ;;
+        q|Q)
+            echo ""
+            printf "${CYAN}Goodbye!${NC}\n"
+            echo ""
+            exit 0
+            ;;
+        *)
+            echo ""
+            printf "${RED}Invalid option${NC}\n"
+            ;;
+    esac
+    
+    echo ""
+    printf "${YELLOW}Press ENTER to continue...${NC}\n"
+    read
+done