@mc1global/opencode-jarvis 0.13.0 → 0.13.1

package/scripts/migration/workspace_structure_migration.py

@@ -0,0 +1,740 @@
+#!/usr/bin/env python3
+"""
+Workspace Structure Migration Script
+
+Migrates existing workspaces from centralized configuration to new DDD-compliant structure:
+workspace/{name}/config|db|obsidian
+
+Author: JARVIS Framework
+Version: 1.0.0
+Date: 2026-01-23
+"""
+
+import os
+import sys
+import json
+import shutil
+import sqlite3
+from pathlib import Path
+from datetime import datetime
+from typing import Dict, List, Any, Optional
+import yaml
+
+# Add project root to path
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from src.shared.paths import PathRegistry
+from src.domain.governance.path_keys import PathKey
+
+
+class WorkspaceMigrator:
+    """Migrates workspace structure to new config|db|obsidian layout"""
+
+    def __init__(self, dry_run: bool = True, backup: bool = True):
+        self.dry_run = dry_run
+        self.backup = backup
+        self.project_root = Path(__file__).parent.parent.parent
+        self.migration_log = []
+
+        # Migration timestamp
+        self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+
+        # Paths
+        self.workspace_dir = self.project_root / "workspace"
+        self.config_dir = self.project_root / "config"
+        self.data_dir = self.project_root / "data"
+        self.backup_dir = self.project_root / f"backup_migration_{self.timestamp}"
+
+    def log(self, message: str, level: str = "INFO"):
+        """Log migration message"""
+        timestamp = datetime.now().strftime("%H:%M:%S")
+        log_entry = f"[{timestamp}] {level}: {message}"
+        print(log_entry)
+        self.migration_log.append(log_entry)
+
+    def create_backup(self):
+        """Create full backup before migration"""
+        if not self.backup:
+            return
+
+        self.log("Creating full backup before migration...")
+
+        try:
+            # Create backup directory
+            self.backup_dir.mkdir(exist_ok=True)
+
+            # Backup workspace directory
+            if self.workspace_dir.exists():
+                shutil.copytree(self.workspace_dir, self.backup_dir / "workspace")
+                self.log(
+                    f"Backed up workspace directory to {self.backup_dir / 'workspace'}"
+                )
+
+            # Backup config files
+            config_backup = self.backup_dir / "config"
+            config_backup.mkdir(exist_ok=True)
+
+            for config_file in ["rag.yaml", "workspace_paths.yaml"]:
+                src = self.config_dir / config_file
+                if src.exists():
+                    shutil.copy2(src, config_backup / config_file)
+                    self.log(f"Backed up {config_file}")
+
+            # Backup kanban databases
+            kanban_backup = self.backup_dir / "data" / "kanban"
+            kanban_backup.mkdir(parents=True, exist_ok=True)
+
+            kanban_dir = self.data_dir / "kanban"
+            if kanban_dir.exists():
+                for db_file in kanban_dir.glob("*.db"):
+                    shutil.copy2(db_file, kanban_backup / db_file.name)
+                    self.log(f"Backed up {db_file.name}")
+
+            # Backup workspace registry
+            registry_file = self.workspace_dir / ".workspace_registry.json"
+            if registry_file.exists():
+                shutil.copy2(registry_file, self.backup_dir / "workspace_registry.json")
+                self.log("Backed up workspace registry")
+
+            self.log(f"✅ Full backup created at: {self.backup_dir}")
+
+        except Exception as e:
+            self.log(f"❌ Backup failed: {e}", "ERROR")
+            raise
+
+    def load_current_workspaces(self) -> Dict[str, Any]:
+        """Load current workspace configurations"""
+        workspaces = {}
+
+        # Load from workspace registry
+        registry_file = self.workspace_dir / ".workspace_registry.json"
+        if registry_file.exists():
+            with open(registry_file, "r") as f:
+                registry = json.load(f)
+                for name, data in registry.items():
+                    workspaces[name] = {
+                        "name": name,
+                        "path": data["path"],
+                        "context_type": data["context_type"],
+                        "created": data["created"],
+                        "external_path": data.get("external_path"),
+                    }
+
+        # Load RAG configurations
+        rag_config_file = self.config_dir / "rag.yaml"
+        if rag_config_file.exists():
+            with open(rag_config_file, "r") as f:
+                rag_config = yaml.safe_load(f)
+                rag_workspaces = rag_config.get("rag", {}).get("workspaces", {})
+
+                for name, rag_data in rag_workspaces.items():
+                    if name in workspaces:
+                        workspaces[name]["rag_config"] = rag_data
+                    else:
+                        # RAG-only workspace (might be orphaned)
+                        self.log(
+                            f"⚠️  Found RAG config for unknown workspace: {name}", "WARN"
+                        )
+
+        # Load workspace paths
+        paths_config_file = self.config_dir / "workspace_paths.yaml"
+        if paths_config_file.exists():
+            with open(paths_config_file, "r") as f:
+                paths_config = yaml.safe_load(f)
+                workspace_paths = paths_config.get("workspaces", {})
+
+                for name, path_data in workspace_paths.items():
+                    if name in workspaces:
+                        workspaces[name]["paths_config"] = path_data
+                    else:
+                        # Paths-only workspace (might be orphaned)
+                        self.log(
+                            f"⚠️  Found paths config for unknown workspace: {name}",
+                            "WARN",
+                        )
+
+        self.log(f"Loaded {len(workspaces)} workspaces from current configuration")
+        return workspaces
+
+    def create_workspace_structure(self, workspace_name: str, workspace_path: Path):
+        """Create new config|db|obsidian structure for workspace"""
+
+        # Create main directories
+        directories = [
+            workspace_path / "config",
+            workspace_path / "db",
+            workspace_path / "obsidian",
+            workspace_path / "obsidian" / "Cards",
+            workspace_path / "obsidian" / "Epics",
+            workspace_path / "obsidian" / "Artifacts",
+            workspace_path / "obsidian" / "Sessions",
+            workspace_path / "obsidian" / ".obsidian",
+        ]
+
+        for directory in directories:
+            if self.dry_run:
+                self.log(f"DRY RUN: Would create directory {directory}")
+            else:
+                directory.mkdir(parents=True, exist_ok=True)
+                self.log(f"Created directory: {directory}")
+
+    def generate_workspace_config(
+        self, workspace_data: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """Generate workspace.yaml configuration"""
+
+        config = {
+            "workspace": {
+                "name": workspace_data["name"],
+                "type": workspace_data.get("context_type", "unknown"),
+                "created": workspace_data.get("created", datetime.now().isoformat()),
+                "version": "1.0.0",
+                "migrated": datetime.now().isoformat(),
+            }
+        }
+
+        # Add external path if exists
+        if workspace_data.get("external_path"):
+            config["workspace"]["paths"] = {
+                "external_project": workspace_data["external_path"]
+            }
+
+        # Add paths config if exists
+        if workspace_data.get("paths_config"):
+            paths_data = workspace_data["paths_config"]
+            if "paths" not in config["workspace"]:
+                config["workspace"]["paths"] = {}
+            config["workspace"]["paths"].update(
+                {
+                    "domain_map": paths_data.get("domain_map", "domain_map.json"),
+                    "description": paths_data.get("description", ""),
+                }
+            )
+
+        # Add settings
+        config["workspace"]["settings"] = {
+            "auto_reindex": False,
+            "auto_backup": True,
+            "obsidian_sync": True,
+        }
+
+        return config
+
+    def generate_rag_config(
+        self, workspace_data: Dict[str, Any]
+    ) -> Optional[Dict[str, Any]]:
+        """Generate workspace-local rag.yaml configuration"""
+
+        rag_data = workspace_data.get("rag_config")
+        if not rag_data:
+            return None
+
+        config = {
+            "rag": {
+                "enabled": rag_data.get("enabled", True),
+                "indexing": {
+                    "base_path": rag_data.get("base_path", "."),
+                    "include_paths": rag_data.get("index_paths", ["src/", "docs/"]),
+                    "exclude_patterns": [
+                        "*.pyc",
+                        "__pycache__/",
+                        ".git/",
+                        "node_modules/",
+                        ".venv/",
+                        "*.log",
+                    ],
+                },
+                "chunking": {"chunk_size": 1000, "chunk_overlap": 200},
+                "vectorstore": {
+                    "type": "chroma",
+                    "path": "db/chroma_db",
+                    "collection_name": f"{workspace_data['name']}_docs",
+                },
+            }
+        }
+
+        return config
+
+    def generate_kanban_config(self, workspace_name: str) -> Dict[str, Any]:
+        """Generate kanban.yaml configuration"""
+
+        config = {
+            "kanban": {
+                "database": "db/kanban.db",
+                "board": {
+                    "columns": [
+                        "backlog",
+                        "grooming",
+                        "ready",
+                        "doing",
+                        "review",
+                        "tested",
+                        "done",
+                    ],
+                    "wip_limits": {"grooming": 5, "ready": 10, "doing": 3, "review": 5},
+                },
+                "sprint": {"default_duration_days": 14, "auto_create": False},
+                "metrics": {
+                    "track_velocity": True,
+                    "track_cycle_time": True,
+                    "track_token_usage": True,
+                },
+            }
+        }
+
+        return config
+
+    def generate_obsidian_config(self, workspace_name: str) -> Dict[str, Any]:
+        """Generate obsidian.yaml configuration"""
+
+        config = {
+            "obsidian": {
+                "enabled": True,
+                "vault_path": "obsidian",
+                "sync": {
+                    "cards_to_notes": True,
+                    "epics_to_notes": True,
+                    "artifacts_linking": True,
+                },
+                "templates": {
+                    "card_template": "templates/card.md",
+                    "epic_template": "templates/epic.md",
+                },
+                "plugins": [
+                    "obsidian-kanban",
+                    "obsidian-dataview",
+                    "obsidian-templater",
+                ],
+            }
+        }
+
+        return config
+
+    def migrate_kanban_database(self, workspace_name: str, workspace_path: Path):
+        """Migrate Kanban database from centralized to workspace-local"""
+
+        # Source database
+        src_db = self.data_dir / "kanban" / f"{workspace_name}_kanban.db"
+
+        # Destination database
+        dest_db = workspace_path / "db" / "kanban.db"
+
+        if not src_db.exists():
+            self.log(f"⚠️  No Kanban database found for {workspace_name}")
+            return
+
+        if self.dry_run:
+            self.log(f"DRY RUN: Would migrate {src_db} → {dest_db}")
+        else:
+            try:
+                # Copy database
+                shutil.copy2(src_db, dest_db)
+                self.log(f"✅ Migrated Kanban database: {src_db} → {dest_db}")
+
+                # Verify database integrity
+                with sqlite3.connect(dest_db) as conn:
+                    cursor = conn.cursor()
+                    cursor.execute("PRAGMA integrity_check")
+                    result = cursor.fetchone()
+                    if result[0] != "ok":
+                        raise Exception(f"Database integrity check failed: {result[0]}")
+
+                self.log(f"✅ Database integrity verified for {workspace_name}")
+
+            except Exception as e:
+                self.log(
+                    f"❌ Failed to migrate Kanban database for {workspace_name}: {e}",
+                    "ERROR",
+                )
+                raise
+
+    def migrate_other_databases(self, workspace_name: str, workspace_path: Path):
+        """Migrate other databases (context_memory, etc.) to workspace db/ directory"""
+
+        workspace_src = Path(self.workspace_dir / workspace_name)
+
+        # Find existing databases in workspace
+        db_files = []
+        if workspace_src.exists():
+            db_files = list(workspace_src.glob("*.db"))
+
+        for db_file in db_files:
+            dest_db = workspace_path / "db" / db_file.name
+
+            if self.dry_run:
+                self.log(f"DRY RUN: Would migrate {db_file} → {dest_db}")
+            else:
+                try:
+                    shutil.copy2(db_file, dest_db)
+                    self.log(f"✅ Migrated database: {db_file} → {dest_db}")
+                except Exception as e:
+                    self.log(f"⚠️  Failed to migrate {db_file}: {e}", "WARN")
+
+    def migrate_chroma_db(self, workspace_name: str, workspace_path: Path):
+        """Migrate ChromaDB vector store to workspace db/ directory"""
+
+        workspace_src = Path(self.workspace_dir / workspace_name)
+        chroma_src = workspace_src / "chroma_db"
+
+        if not chroma_src.exists():
+            self.log(f"No ChromaDB found for {workspace_name}")
+            return
+
+        chroma_dest = workspace_path / "db" / "chroma_db"
+
+        if self.dry_run:
+            self.log(f"DRY RUN: Would migrate {chroma_src} → {chroma_dest}")
+        else:
+            try:
+                shutil.copytree(chroma_src, chroma_dest)
+                self.log(f"✅ Migrated ChromaDB: {chroma_src} → {chroma_dest}")
+            except Exception as e:
+                self.log(
+                    f"⚠️  Failed to migrate ChromaDB for {workspace_name}: {e}", "WARN"
+                )
+
+    def write_config_file(self, config: Dict[str, Any], file_path: Path):
+        """Write configuration to YAML file"""
+
+        if self.dry_run:
+            self.log(f"DRY RUN: Would write config to {file_path}")
+            return
+
+        try:
+            with open(file_path, "w") as f:
+                yaml.dump(
+                    config, f, default_flow_style=False, sort_keys=False, indent=2
+                )
+            self.log(f"✅ Created config file: {file_path}")
+        except Exception as e:
+            self.log(f"❌ Failed to write config {file_path}: {e}", "ERROR")
+            raise
+
+    def migrate_workspace(self, workspace_name: str, workspace_data: Dict[str, Any]):
+        """Migrate a single workspace to new structure"""
+
+        self.log(f"\n🔄 Migrating workspace: {workspace_name}")
+
+        workspace_path = Path(workspace_data["path"])
+
+        # Special handling for JARVIS (meta-workspace)
+        is_jarvis = workspace_name == "JARVIS"
+
+        try:
+            # 1. Create new directory structure
+            self.create_workspace_structure(workspace_name, workspace_path)
+
+            # 2. Generate and write configuration files
+            workspace_config = self.generate_workspace_config(workspace_data)
+            if is_jarvis:
+                workspace_config["workspace"]["type"] = "meta-workspace"
+                workspace_config["workspace"]["meta"] = {
+                    "manage_other_workspaces": True,
+                    "centralized_coordination": True,
+                    "cross_workspace_visibility": True,
+                }
+
+            self.write_config_file(
+                workspace_config, workspace_path / "config" / "workspace.yaml"
+            )
+
+            # 3. Generate RAG config if exists
+            rag_config = self.generate_rag_config(workspace_data)
+            if rag_config:
+                self.write_config_file(
+                    rag_config, workspace_path / "config" / "rag.yaml"
+                )
+
+            # 4. Generate Kanban config
+            kanban_config = self.generate_kanban_config(workspace_name)
+            self.write_config_file(
+                kanban_config, workspace_path / "config" / "kanban.yaml"
+            )
+
+            # 5. Generate Obsidian config
+            obsidian_config = self.generate_obsidian_config(workspace_name)
+            self.write_config_file(
+                obsidian_config, workspace_path / "config" / "obsidian.yaml"
+            )
+
+            # 6. Migrate databases
+            self.migrate_kanban_database(workspace_name, workspace_path)
+            self.migrate_other_databases(workspace_name, workspace_path)
+            self.migrate_chroma_db(workspace_name, workspace_path)
+
+            self.log(f"✅ Successfully migrated workspace: {workspace_name}")
+
+        except Exception as e:
+            self.log(f"❌ Failed to migrate workspace {workspace_name}: {e}", "ERROR")
+            raise
+
+    def cleanup_centralized_configs(self, workspaces: Dict[str, Any]):
+        """Clean up centralized configuration files after successful migration"""
+
+        if self.dry_run:
+            self.log("DRY RUN: Would clean up centralized configurations")
+            return
+
+        self.log("\n🧹 Cleaning up centralized configurations...")
+
+        try:
+            # Remove workspace-specific RAG configurations
+            rag_config_file = self.config_dir / "rag.yaml"
+            if rag_config_file.exists():
+                with open(rag_config_file, "r") as f:
+                    rag_config = yaml.safe_load(f)
+
+                # Remove workspace-specific configurations
+                if "rag" in rag_config and "workspaces" in rag_config["rag"]:
+                    del rag_config["rag"]["workspaces"]
+
+                    # Write back cleaned config
+                    with open(rag_config_file, "w") as f:
+                        yaml.dump(
+                            rag_config,
+                            f,
+                            default_flow_style=False,
+                            sort_keys=False,
+                            indent=2,
+                        )
+
+                    self.log("✅ Cleaned workspace configs from rag.yaml")
+
+            # Remove workspace paths (keep only framework-level paths)
+            paths_config_file = self.config_dir / "workspace_paths.yaml"
+            if paths_config_file.exists():
+                # Create backup and remove workspace-specific paths
+                backup_paths = paths_config_file.with_suffix(
+                    f".yaml.backup_{self.timestamp}"
+                )
+                shutil.copy2(paths_config_file, backup_paths)
+
+                # Keep only framework-level configuration
+                framework_config = {
+                    "framework": {
+                        "name": "JARVIS",
+                        "version": "1.0.0",
+                        "migrated": datetime.now().isoformat(),
+                    }
+                }
+
+                with open(paths_config_file, "w") as f:
+                    yaml.dump(
+                        framework_config,
+                        f,
+                        default_flow_style=False,
+                        sort_keys=False,
+                        indent=2,
+                    )
+
+                self.log(f"✅ Cleaned workspace_paths.yaml (backup: {backup_paths})")
+
+        except Exception as e:
+            self.log(f"⚠️  Error during cleanup: {e}", "WARN")
+
+    def generate_migration_report(self, workspaces: Dict[str, Any]):
+        """Generate migration report"""
+
+        report_path = self.project_root / f"migration_report_{self.timestamp}.md"
+
+        report_content = f"""# Workspace Structure Migration Report
+
+**Date**: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}  
+**Mode**: {"DRY RUN" if self.dry_run else "LIVE MIGRATION"}  
+**Backup**: {"YES" if self.backup else "NO"}
+
+## Summary
+
+- **Total Workspaces**: {len(workspaces)}
+- **Backup Location**: {self.backup_dir if self.backup else "N/A"}
+- **Migration Status**: {"SIMULATED" if self.dry_run else "COMPLETED"}
+
+## Migrated Workspaces
+
+"""
+
+        for name, data in workspaces.items():
+            report_content += f"### {name}\n"
+            report_content += f"- **Type**: {data.get('context_type', 'unknown')}\n"
+            report_content += f"- **Path**: {data['path']}\n"
+            report_content += (
+                f"- **External Path**: {data.get('external_path', 'N/A')}\n"
+            )
+            report_content += (
+                f"- **Has RAG Config**: {'YES' if data.get('rag_config') else 'NO'}\n"
+            )
+            report_content += f"- **Has Paths Config**: {'YES' if data.get('paths_config') else 'NO'}\n\n"
+
+        report_content += f"""
+## Migration Log
+
+```
+{"".join([f"{line}\n" for line in self.migration_log])}
+```
+
+## New Workspace Structure
+
+Each workspace now has the following structure:
+
+```
+workspace/{{name}}/
+├── config/
+│   ├── workspace.yaml    # Workspace metadata and settings
+│   ├── rag.yaml         # RAG configuration (if applicable)
+│   ├── kanban.yaml      # Kanban board settings
+│   └── obsidian.yaml    # Obsidian vault configuration
+├── db/
+│   ├── kanban.db        # Kanban database
+│   ├── context_memory.db # Context and sessions
+│   └── chroma_db/       # Vector embeddings
+├── obsidian/
+│   ├── Cards/           # Kanban cards as notes
+│   ├── Epics/           # Epic documentation
+│   ├── Artifacts/       # Linked artifacts
+│   ├── Sessions/        # Session notes
+│   └── .obsidian/       # Obsidian settings
+└── [existing directories] # artifacts, docs, logs, etc.
+```
+
+## Next Steps
+
+1. **Verify Migration**: Check that all data was migrated correctly
+2. **Update Systems**: Update all managers to use new workspace-local configs
+3. **Test Functionality**: Ensure all features work with new structure
+4. **Clean Old Files**: Remove centralized configuration files after verification
+5. **Update Documentation**: Update all documentation to reflect new structure
+
+## Rollback Instructions
+
+If rollback is needed:
+
+1. Stop all JARVIS services
+2. Restore from backup: `cp -r {self.backup_dir}/* .`
+3. Restart services
+4. Verify functionality
+
+---
+
+**Migration Tool**: `scripts/migration/workspace_structure_migration.py`  
+**Report Generated**: {datetime.now().isoformat()}
+"""
+
+        if not self.dry_run:
+            with open(report_path, "w") as f:
+                f.write(report_content)
+            self.log(f"✅ Migration report saved to: {report_path}")
+        else:
+            self.log(f"DRY RUN: Would save migration report to: {report_path}")
+
+        return report_content
+
+    def run_migration(self):
+        """Run the complete migration process"""
+
+        self.log("🚀 Starting Workspace Structure Migration")
+        self.log(f"Mode: {'DRY RUN' if self.dry_run else 'LIVE MIGRATION'}")
+        self.log(f"Backup: {'ENABLED' if self.backup else 'DISABLED'}")
+
+        try:
+            # 1. Create backup
+            if self.backup:
+                self.create_backup()
+
+            # 2. Load current workspaces
+            workspaces = self.load_current_workspaces()
+
+            if not workspaces:
+                self.log("❌ No workspaces found to migrate")
+                return False
+
+            # 3. Migrate each workspace
+            for workspace_name, workspace_data in workspaces.items():
+                self.migrate_workspace(workspace_name, workspace_data)
+
+            # 4. Clean up centralized configs (only in live mode)
+            if not self.dry_run:
+                self.cleanup_centralized_configs(workspaces)
+
+            # 5. Generate migration report
+            self.generate_migration_report(workspaces)
+
+            self.log(
+                f"\n✅ Migration {'simulation' if self.dry_run else 'completed'} successfully!"
+            )
+            self.log(f"📊 Migrated {len(workspaces)} workspaces")
+
+            if self.backup:
+                self.log(f"💾 Backup available at: {self.backup_dir}")
+
+            return True
+
+        except Exception as e:
+            self.log(f"❌ Migration failed: {e}", "ERROR")
+            if self.backup:
+                self.log(f"💾 Restore from backup: {self.backup_dir}")
+            raise
+
+
+def main():
+    """Main entry point"""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Migrate JARVIS workspaces to new config|db|obsidian structure"
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        default=True,
+        help="Simulate migration without making changes (default: True)",
+    )
+    parser.add_argument(
+        "--live", action="store_true", help="Run live migration (overrides --dry-run)"
+    )
+    parser.add_argument(
+        "--no-backup",
+        action="store_true",
+        help="Skip backup creation (not recommended)",
+    )
+
+    args = parser.parse_args()
+
+    # Determine run mode
+    dry_run = not args.live  # Default to dry-run unless --live is specified
+    backup = not args.no_backup
+
+    print(f"""
+╔══════════════════════════════════════════════════════════════════════════════╗
+║                    JARVIS Workspace Structure Migration                      ║
+║                                                                              ║
+║  This tool migrates workspaces to the new config|db|obsidian structure      ║
+║                                                                              ║
+║  Mode: {"🧪 DRY RUN (simulation only)" if dry_run else "🔴 LIVE MIGRATION (makes changes)"}                    ║
+║  Backup: {"✅ ENABLED" if backup else "❌ DISABLED (dangerous!)"}                                      ║
+╚══════════════════════════════════════════════════════════════════════════════╝
+""")
+
+    if not dry_run:
+        confirm = input(
+            "\n⚠️  You are about to run LIVE MIGRATION. This will modify your workspace structure.\n"
+            "Type 'MIGRATE' to confirm: "
+        )
+        if confirm != "MIGRATE":
+            print("❌ Migration cancelled.")
+            return 1
+
+    try:
+        migrator = WorkspaceMigrator(dry_run=dry_run, backup=backup)
+        success = migrator.run_migration()
+        return 0 if success else 1
+
+    except KeyboardInterrupt:
+        print("\n❌ Migration cancelled by user.")
+        return 1
+    except Exception as e:
+        print(f"\n❌ Migration failed: {e}")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())