@mc1global/opencode-jarvis 0.13.1 → 0.14.0

package/scripts/migration/workspace_structure_migration.py

@@ -1,740 +0,0 @@
-#!/usr/bin/env python3
-"""
-Workspace Structure Migration Script
-
-Migrates existing workspaces from centralized configuration to new DDD-compliant structure:
-workspace/{name}/config|db|obsidian
-
-Author: JARVIS Framework
-Version: 1.0.0
-Date: 2026-01-23
-"""
-
-import os
-import sys
-import json
-import shutil
-import sqlite3
-from pathlib import Path
-from datetime import datetime
-from typing import Dict, List, Any, Optional
-import yaml
-
-# Add project root to path
-sys.path.insert(0, str(Path(__file__).parent.parent.parent))
-
-from src.shared.paths import PathRegistry
-from src.domain.governance.path_keys import PathKey
-
-
-class WorkspaceMigrator:
-    """Migrates workspace structure to new config|db|obsidian layout"""
-
-    def __init__(self, dry_run: bool = True, backup: bool = True):
-        self.dry_run = dry_run
-        self.backup = backup
-        self.project_root = Path(__file__).parent.parent.parent
-        self.migration_log = []
-
-        # Migration timestamp
-        self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-
-        # Paths
-        self.workspace_dir = self.project_root / "workspace"
-        self.config_dir = self.project_root / "config"
-        self.data_dir = self.project_root / "data"
-        self.backup_dir = self.project_root / f"backup_migration_{self.timestamp}"
-
-    def log(self, message: str, level: str = "INFO"):
-        """Log migration message"""
-        timestamp = datetime.now().strftime("%H:%M:%S")
-        log_entry = f"[{timestamp}] {level}: {message}"
-        print(log_entry)
-        self.migration_log.append(log_entry)
-
-    def create_backup(self):
-        """Create full backup before migration"""
-        if not self.backup:
-            return
-
-        self.log("Creating full backup before migration...")
-
-        try:
-            # Create backup directory
-            self.backup_dir.mkdir(exist_ok=True)
-
-            # Backup workspace directory
-            if self.workspace_dir.exists():
-                shutil.copytree(self.workspace_dir, self.backup_dir / "workspace")
-                self.log(
-                    f"Backed up workspace directory to {self.backup_dir / 'workspace'}"
-                )
-
-            # Backup config files
-            config_backup = self.backup_dir / "config"
-            config_backup.mkdir(exist_ok=True)
-
-            for config_file in ["rag.yaml", "workspace_paths.yaml"]:
-                src = self.config_dir / config_file
-                if src.exists():
-                    shutil.copy2(src, config_backup / config_file)
-                    self.log(f"Backed up {config_file}")
-
-            # Backup kanban databases
-            kanban_backup = self.backup_dir / "data" / "kanban"
-            kanban_backup.mkdir(parents=True, exist_ok=True)
-
-            kanban_dir = self.data_dir / "kanban"
-            if kanban_dir.exists():
-                for db_file in kanban_dir.glob("*.db"):
-                    shutil.copy2(db_file, kanban_backup / db_file.name)
-                    self.log(f"Backed up {db_file.name}")
-
-            # Backup workspace registry
-            registry_file = self.workspace_dir / ".workspace_registry.json"
-            if registry_file.exists():
-                shutil.copy2(registry_file, self.backup_dir / "workspace_registry.json")
-                self.log("Backed up workspace registry")
-
-            self.log(f"✅ Full backup created at: {self.backup_dir}")
-
-        except Exception as e:
-            self.log(f"❌ Backup failed: {e}", "ERROR")
-            raise
-
-    def load_current_workspaces(self) -> Dict[str, Any]:
-        """Load current workspace configurations"""
-        workspaces = {}
-
-        # Load from workspace registry
-        registry_file = self.workspace_dir / ".workspace_registry.json"
-        if registry_file.exists():
-            with open(registry_file, "r") as f:
-                registry = json.load(f)
-                for name, data in registry.items():
-                    workspaces[name] = {
-                        "name": name,
-                        "path": data["path"],
-                        "context_type": data["context_type"],
-                        "created": data["created"],
-                        "external_path": data.get("external_path"),
-                    }
-
-        # Load RAG configurations
-        rag_config_file = self.config_dir / "rag.yaml"
-        if rag_config_file.exists():
-            with open(rag_config_file, "r") as f:
-                rag_config = yaml.safe_load(f)
-                rag_workspaces = rag_config.get("rag", {}).get("workspaces", {})
-
-                for name, rag_data in rag_workspaces.items():
-                    if name in workspaces:
-                        workspaces[name]["rag_config"] = rag_data
-                    else:
-                        # RAG-only workspace (might be orphaned)
-                        self.log(
-                            f"⚠️  Found RAG config for unknown workspace: {name}", "WARN"
-                        )
-
-        # Load workspace paths
-        paths_config_file = self.config_dir / "workspace_paths.yaml"
-        if paths_config_file.exists():
-            with open(paths_config_file, "r") as f:
-                paths_config = yaml.safe_load(f)
-                workspace_paths = paths_config.get("workspaces", {})
-
-                for name, path_data in workspace_paths.items():
-                    if name in workspaces:
-                        workspaces[name]["paths_config"] = path_data
-                    else:
-                        # Paths-only workspace (might be orphaned)
-                        self.log(
-                            f"⚠️  Found paths config for unknown workspace: {name}",
-                            "WARN",
-                        )
-
-        self.log(f"Loaded {len(workspaces)} workspaces from current configuration")
-        return workspaces
-
-    def create_workspace_structure(self, workspace_name: str, workspace_path: Path):
-        """Create new config|db|obsidian structure for workspace"""
-
-        # Create main directories
-        directories = [
-            workspace_path / "config",
-            workspace_path / "db",
-            workspace_path / "obsidian",
-            workspace_path / "obsidian" / "Cards",
-            workspace_path / "obsidian" / "Epics",
-            workspace_path / "obsidian" / "Artifacts",
-            workspace_path / "obsidian" / "Sessions",
-            workspace_path / "obsidian" / ".obsidian",
-        ]
-
-        for directory in directories:
-            if self.dry_run:
-                self.log(f"DRY RUN: Would create directory {directory}")
-            else:
-                directory.mkdir(parents=True, exist_ok=True)
-                self.log(f"Created directory: {directory}")
-
-    def generate_workspace_config(
-        self, workspace_data: Dict[str, Any]
-    ) -> Dict[str, Any]:
-        """Generate workspace.yaml configuration"""
-
-        config = {
-            "workspace": {
-                "name": workspace_data["name"],
-                "type": workspace_data.get("context_type", "unknown"),
-                "created": workspace_data.get("created", datetime.now().isoformat()),
-                "version": "1.0.0",
-                "migrated": datetime.now().isoformat(),
-            }
-        }
-
-        # Add external path if exists
-        if workspace_data.get("external_path"):
-            config["workspace"]["paths"] = {
-                "external_project": workspace_data["external_path"]
-            }
-
-        # Add paths config if exists
-        if workspace_data.get("paths_config"):
-            paths_data = workspace_data["paths_config"]
-            if "paths" not in config["workspace"]:
-                config["workspace"]["paths"] = {}
-            config["workspace"]["paths"].update(
-                {
-                    "domain_map": paths_data.get("domain_map", "domain_map.json"),
-                    "description": paths_data.get("description", ""),
-                }
-            )
-
-        # Add settings
-        config["workspace"]["settings"] = {
-            "auto_reindex": False,
-            "auto_backup": True,
-            "obsidian_sync": True,
-        }
-
-        return config
-
-    def generate_rag_config(
-        self, workspace_data: Dict[str, Any]
-    ) -> Optional[Dict[str, Any]]:
-        """Generate workspace-local rag.yaml configuration"""
-
-        rag_data = workspace_data.get("rag_config")
-        if not rag_data:
-            return None
-
-        config = {
-            "rag": {
-                "enabled": rag_data.get("enabled", True),
-                "indexing": {
-                    "base_path": rag_data.get("base_path", "."),
-                    "include_paths": rag_data.get("index_paths", ["src/", "docs/"]),
-                    "exclude_patterns": [
-                        "*.pyc",
-                        "__pycache__/",
-                        ".git/",
-                        "node_modules/",
-                        ".venv/",
-                        "*.log",
-                    ],
-                },
-                "chunking": {"chunk_size": 1000, "chunk_overlap": 200},
-                "vectorstore": {
-                    "type": "chroma",
-                    "path": "db/chroma_db",
-                    "collection_name": f"{workspace_data['name']}_docs",
-                },
-            }
-        }
-
-        return config
-
-    def generate_kanban_config(self, workspace_name: str) -> Dict[str, Any]:
-        """Generate kanban.yaml configuration"""
-
-        config = {
-            "kanban": {
-                "database": "db/kanban.db",
-                "board": {
-                    "columns": [
-                        "backlog",
-                        "grooming",
-                        "ready",
-                        "doing",
-                        "review",
-                        "tested",
-                        "done",
-                    ],
-                    "wip_limits": {"grooming": 5, "ready": 10, "doing": 3, "review": 5},
-                },
-                "sprint": {"default_duration_days": 14, "auto_create": False},
-                "metrics": {
-                    "track_velocity": True,
-                    "track_cycle_time": True,
-                    "track_token_usage": True,
-                },
-            }
-        }
-
-        return config
-
-    def generate_obsidian_config(self, workspace_name: str) -> Dict[str, Any]:
-        """Generate obsidian.yaml configuration"""
-
-        config = {
-            "obsidian": {
-                "enabled": True,
-                "vault_path": "obsidian",
-                "sync": {
-                    "cards_to_notes": True,
-                    "epics_to_notes": True,
-                    "artifacts_linking": True,
-                },
-                "templates": {
-                    "card_template": "templates/card.md",
-                    "epic_template": "templates/epic.md",
-                },
-                "plugins": [
-                    "obsidian-kanban",
-                    "obsidian-dataview",
-                    "obsidian-templater",
-                ],
-            }
-        }
-
-        return config
-
-    def migrate_kanban_database(self, workspace_name: str, workspace_path: Path):
-        """Migrate Kanban database from centralized to workspace-local"""
-
-        # Source database
-        src_db = self.data_dir / "kanban" / f"{workspace_name}_kanban.db"
-
-        # Destination database
-        dest_db = workspace_path / "db" / "kanban.db"
-
-        if not src_db.exists():
-            self.log(f"⚠️  No Kanban database found for {workspace_name}")
-            return
-
-        if self.dry_run:
-            self.log(f"DRY RUN: Would migrate {src_db} → {dest_db}")
-        else:
-            try:
-                # Copy database
-                shutil.copy2(src_db, dest_db)
-                self.log(f"✅ Migrated Kanban database: {src_db} → {dest_db}")
-
-                # Verify database integrity
-                with sqlite3.connect(dest_db) as conn:
-                    cursor = conn.cursor()
-                    cursor.execute("PRAGMA integrity_check")
-                    result = cursor.fetchone()
-                    if result[0] != "ok":
-                        raise Exception(f"Database integrity check failed: {result[0]}")
-
-                self.log(f"✅ Database integrity verified for {workspace_name}")
-
-            except Exception as e:
-                self.log(
-                    f"❌ Failed to migrate Kanban database for {workspace_name}: {e}",
-                    "ERROR",
-                )
-                raise
-
-    def migrate_other_databases(self, workspace_name: str, workspace_path: Path):
-        """Migrate other databases (context_memory, etc.) to workspace db/ directory"""
-
-        workspace_src = Path(self.workspace_dir / workspace_name)
-
-        # Find existing databases in workspace
-        db_files = []
-        if workspace_src.exists():
-            db_files = list(workspace_src.glob("*.db"))
-
-        for db_file in db_files:
-            dest_db = workspace_path / "db" / db_file.name
-
-            if self.dry_run:
-                self.log(f"DRY RUN: Would migrate {db_file} → {dest_db}")
-            else:
-                try:
-                    shutil.copy2(db_file, dest_db)
-                    self.log(f"✅ Migrated database: {db_file} → {dest_db}")
-                except Exception as e:
-                    self.log(f"⚠️  Failed to migrate {db_file}: {e}", "WARN")
-
-    def migrate_chroma_db(self, workspace_name: str, workspace_path: Path):
-        """Migrate ChromaDB vector store to workspace db/ directory"""
-
-        workspace_src = Path(self.workspace_dir / workspace_name)
-        chroma_src = workspace_src / "chroma_db"
-
-        if not chroma_src.exists():
-            self.log(f"No ChromaDB found for {workspace_name}")
-            return
-
-        chroma_dest = workspace_path / "db" / "chroma_db"
-
-        if self.dry_run:
-            self.log(f"DRY RUN: Would migrate {chroma_src} → {chroma_dest}")
-        else:
-            try:
-                shutil.copytree(chroma_src, chroma_dest)
-                self.log(f"✅ Migrated ChromaDB: {chroma_src} → {chroma_dest}")
-            except Exception as e:
-                self.log(
-                    f"⚠️  Failed to migrate ChromaDB for {workspace_name}: {e}", "WARN"
-                )
-
-    def write_config_file(self, config: Dict[str, Any], file_path: Path):
-        """Write configuration to YAML file"""
-
-        if self.dry_run:
-            self.log(f"DRY RUN: Would write config to {file_path}")
-            return
-
-        try:
-            with open(file_path, "w") as f:
-                yaml.dump(
-                    config, f, default_flow_style=False, sort_keys=False, indent=2
-                )
-            self.log(f"✅ Created config file: {file_path}")
-        except Exception as e:
-            self.log(f"❌ Failed to write config {file_path}: {e}", "ERROR")
-            raise
-
-    def migrate_workspace(self, workspace_name: str, workspace_data: Dict[str, Any]):
-        """Migrate a single workspace to new structure"""
-
-        self.log(f"\n🔄 Migrating workspace: {workspace_name}")
-
-        workspace_path = Path(workspace_data["path"])
-
-        # Special handling for JARVIS (meta-workspace)
-        is_jarvis = workspace_name == "JARVIS"
-
-        try:
-            # 1. Create new directory structure
-            self.create_workspace_structure(workspace_name, workspace_path)
-
-            # 2. Generate and write configuration files
-            workspace_config = self.generate_workspace_config(workspace_data)
-            if is_jarvis:
-                workspace_config["workspace"]["type"] = "meta-workspace"
-                workspace_config["workspace"]["meta"] = {
-                    "manage_other_workspaces": True,
-                    "centralized_coordination": True,
-                    "cross_workspace_visibility": True,
-                }
-
-            self.write_config_file(
-                workspace_config, workspace_path / "config" / "workspace.yaml"
-            )
-
-            # 3. Generate RAG config if exists
-            rag_config = self.generate_rag_config(workspace_data)
-            if rag_config:
-                self.write_config_file(
-                    rag_config, workspace_path / "config" / "rag.yaml"
-                )
-
-            # 4. Generate Kanban config
-            kanban_config = self.generate_kanban_config(workspace_name)
-            self.write_config_file(
-                kanban_config, workspace_path / "config" / "kanban.yaml"
-            )
-
-            # 5. Generate Obsidian config
-            obsidian_config = self.generate_obsidian_config(workspace_name)
-            self.write_config_file(
-                obsidian_config, workspace_path / "config" / "obsidian.yaml"
-            )
-
-            # 6. Migrate databases
-            self.migrate_kanban_database(workspace_name, workspace_path)
-            self.migrate_other_databases(workspace_name, workspace_path)
-            self.migrate_chroma_db(workspace_name, workspace_path)
-
-            self.log(f"✅ Successfully migrated workspace: {workspace_name}")
-
-        except Exception as e:
-            self.log(f"❌ Failed to migrate workspace {workspace_name}: {e}", "ERROR")
-            raise
-
-    def cleanup_centralized_configs(self, workspaces: Dict[str, Any]):
-        """Clean up centralized configuration files after successful migration"""
-
-        if self.dry_run:
-            self.log("DRY RUN: Would clean up centralized configurations")
-            return
-
-        self.log("\n🧹 Cleaning up centralized configurations...")
-
-        try:
-            # Remove workspace-specific RAG configurations
-            rag_config_file = self.config_dir / "rag.yaml"
-            if rag_config_file.exists():
-                with open(rag_config_file, "r") as f:
-                    rag_config = yaml.safe_load(f)
-
-                # Remove workspace-specific configurations
-                if "rag" in rag_config and "workspaces" in rag_config["rag"]:
-                    del rag_config["rag"]["workspaces"]
-
-                    # Write back cleaned config
-                    with open(rag_config_file, "w") as f:
-                        yaml.dump(
-                            rag_config,
-                            f,
-                            default_flow_style=False,
-                            sort_keys=False,
-                            indent=2,
-                        )
-
-                    self.log("✅ Cleaned workspace configs from rag.yaml")
-
-            # Remove workspace paths (keep only framework-level paths)
-            paths_config_file = self.config_dir / "workspace_paths.yaml"
-            if paths_config_file.exists():
-                # Create backup and remove workspace-specific paths
-                backup_paths = paths_config_file.with_suffix(
-                    f".yaml.backup_{self.timestamp}"
-                )
-                shutil.copy2(paths_config_file, backup_paths)
-
-                # Keep only framework-level configuration
-                framework_config = {
-                    "framework": {
-                        "name": "JARVIS",
-                        "version": "1.0.0",
-                        "migrated": datetime.now().isoformat(),
-                    }
-                }
-
-                with open(paths_config_file, "w") as f:
-                    yaml.dump(
-                        framework_config,
-                        f,
-                        default_flow_style=False,
-                        sort_keys=False,
-                        indent=2,
-                    )
-
-                self.log(f"✅ Cleaned workspace_paths.yaml (backup: {backup_paths})")
-
-        except Exception as e:
-            self.log(f"⚠️  Error during cleanup: {e}", "WARN")
-
-    def generate_migration_report(self, workspaces: Dict[str, Any]):
-        """Generate migration report"""
-
-        report_path = self.project_root / f"migration_report_{self.timestamp}.md"
-
-        report_content = f"""# Workspace Structure Migration Report
-
-**Date**: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}  
-**Mode**: {"DRY RUN" if self.dry_run else "LIVE MIGRATION"}  
-**Backup**: {"YES" if self.backup else "NO"}
-
-## Summary
-
-- **Total Workspaces**: {len(workspaces)}
-- **Backup Location**: {self.backup_dir if self.backup else "N/A"}
-- **Migration Status**: {"SIMULATED" if self.dry_run else "COMPLETED"}
-
-## Migrated Workspaces
-
-"""
-
-        for name, data in workspaces.items():
-            report_content += f"### {name}\n"
-            report_content += f"- **Type**: {data.get('context_type', 'unknown')}\n"
-            report_content += f"- **Path**: {data['path']}\n"
-            report_content += (
-                f"- **External Path**: {data.get('external_path', 'N/A')}\n"
-            )
-            report_content += (
-                f"- **Has RAG Config**: {'YES' if data.get('rag_config') else 'NO'}\n"
-            )
-            report_content += f"- **Has Paths Config**: {'YES' if data.get('paths_config') else 'NO'}\n\n"
-
-        report_content += f"""
-## Migration Log
-
-```
-{"".join([f"{line}\n" for line in self.migration_log])}
-```
-
-## New Workspace Structure
-
-Each workspace now has the following structure:
-
-```
-workspace/{{name}}/
-├── config/
-│   ├── workspace.yaml    # Workspace metadata and settings
-│   ├── rag.yaml         # RAG configuration (if applicable)
-│   ├── kanban.yaml      # Kanban board settings
-│   └── obsidian.yaml    # Obsidian vault configuration
-├── db/
-│   ├── kanban.db        # Kanban database
-│   ├── context_memory.db # Context and sessions
-│   └── chroma_db/       # Vector embeddings
-├── obsidian/
-│   ├── Cards/           # Kanban cards as notes
-│   ├── Epics/           # Epic documentation
-│   ├── Artifacts/       # Linked artifacts
-│   ├── Sessions/        # Session notes
-│   └── .obsidian/       # Obsidian settings
-└── [existing directories] # artifacts, docs, logs, etc.
-```
-
-## Next Steps
-
-1. **Verify Migration**: Check that all data was migrated correctly
-2. **Update Systems**: Update all managers to use new workspace-local configs
-3. **Test Functionality**: Ensure all features work with new structure
-4. **Clean Old Files**: Remove centralized configuration files after verification
-5. **Update Documentation**: Update all documentation to reflect new structure
-
-## Rollback Instructions
-
-If rollback is needed:
-
-1. Stop all JARVIS services
-2. Restore from backup: `cp -r {self.backup_dir}/* .`
-3. Restart services
-4. Verify functionality
-
----
-
-**Migration Tool**: `scripts/migration/workspace_structure_migration.py`  
-**Report Generated**: {datetime.now().isoformat()}
-"""
-
-        if not self.dry_run:
-            with open(report_path, "w") as f:
-                f.write(report_content)
-            self.log(f"✅ Migration report saved to: {report_path}")
-        else:
-            self.log(f"DRY RUN: Would save migration report to: {report_path}")
-
-        return report_content
-
-    def run_migration(self):
-        """Run the complete migration process"""
-
-        self.log("🚀 Starting Workspace Structure Migration")
-        self.log(f"Mode: {'DRY RUN' if self.dry_run else 'LIVE MIGRATION'}")
-        self.log(f"Backup: {'ENABLED' if self.backup else 'DISABLED'}")
-
-        try:
-            # 1. Create backup
-            if self.backup:
-                self.create_backup()
-
-            # 2. Load current workspaces
-            workspaces = self.load_current_workspaces()
-
-            if not workspaces:
-                self.log("❌ No workspaces found to migrate")
-                return False
-
-            # 3. Migrate each workspace
-            for workspace_name, workspace_data in workspaces.items():
-                self.migrate_workspace(workspace_name, workspace_data)
-
-            # 4. Clean up centralized configs (only in live mode)
-            if not self.dry_run:
-                self.cleanup_centralized_configs(workspaces)
-
-            # 5. Generate migration report
-            self.generate_migration_report(workspaces)
-
-            self.log(
-                f"\n✅ Migration {'simulation' if self.dry_run else 'completed'} successfully!"
-            )
-            self.log(f"📊 Migrated {len(workspaces)} workspaces")
-
-            if self.backup:
-                self.log(f"💾 Backup available at: {self.backup_dir}")
-
-            return True
-
-        except Exception as e:
-            self.log(f"❌ Migration failed: {e}", "ERROR")
-            if self.backup:
-                self.log(f"💾 Restore from backup: {self.backup_dir}")
-            raise
-
-
-def main():
-    """Main entry point"""
-    import argparse
-
-    parser = argparse.ArgumentParser(
-        description="Migrate JARVIS workspaces to new config|db|obsidian structure"
-    )
-    parser.add_argument(
-        "--dry-run",
-        action="store_true",
-        default=True,
-        help="Simulate migration without making changes (default: True)",
-    )
-    parser.add_argument(
-        "--live", action="store_true", help="Run live migration (overrides --dry-run)"
-    )
-    parser.add_argument(
-        "--no-backup",
-        action="store_true",
-        help="Skip backup creation (not recommended)",
-    )
-
-    args = parser.parse_args()
-
-    # Determine run mode
-    dry_run = not args.live  # Default to dry-run unless --live is specified
-    backup = not args.no_backup
-
-    print(f"""
-╔══════════════════════════════════════════════════════════════════════════════╗
-║                    JARVIS Workspace Structure Migration                      ║
-║                                                                              ║
-║  This tool migrates workspaces to the new config|db|obsidian structure      ║
-║                                                                              ║
-║  Mode: {"🧪 DRY RUN (simulation only)" if dry_run else "🔴 LIVE MIGRATION (makes changes)"}                    ║
-║  Backup: {"✅ ENABLED" if backup else "❌ DISABLED (dangerous!)"}                                      ║
-╚══════════════════════════════════════════════════════════════════════════════╝
-""")
-
-    if not dry_run:
-        confirm = input(
-            "\n⚠️  You are about to run LIVE MIGRATION. This will modify your workspace structure.\n"
-            "Type 'MIGRATE' to confirm: "
-        )
-        if confirm != "MIGRATE":
-            print("❌ Migration cancelled.")
-            return 1
-
-    try:
-        migrator = WorkspaceMigrator(dry_run=dry_run, backup=backup)
-        success = migrator.run_migration()
-        return 0 if success else 1
-
-    except KeyboardInterrupt:
-        print("\n❌ Migration cancelled by user.")
-        return 1
-    except Exception as e:
-        print(f"\n❌ Migration failed: {e}")
-        return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())