@ngocsangairvds/vsaf 3.2.16 → 4.0.1

package/tools/vds-scripts/memory_orchestrator/src/vds_memory_orchestrator/service/services/memory_export_import.py

@@ -1,1403 +0,0 @@
-"""
-VDS AI Memory Export/Import Service
-
-Handles async export and import operations with job tracking, progress monitoring,
-and format conversion. Uses Celery for background tasks and Redis for real-time
-job status tracking with PostgreSQL for persistence.
-"""
-
-import asyncio
-import contextlib
-import csv
-import json
-import logging
-import mimetypes
-import os
-import uuid
-import zipfile
-from datetime import UTC, datetime, timedelta
-from pathlib import Path
-from typing import Any
-
-import pandas as pd
-import redis
-from celery import Celery
-from fastapi import HTTPException, UploadFile
-from sqlalchemy import func, select
-from sqlalchemy.orm import Session
-
-from vds_memory_orchestrator.service.core.config import get_config
-from vds_memory_orchestrator.service.models.ai_memory_models import Memory
-from vds_memory_orchestrator.service.models.database import ExportImportJob, get_db
-from vds_memory_orchestrator.service.models.export_import_models import (
-    ExportFormat,
-    ExportRequest,
-    ExportSummary,
-    ImportConflictStrategy,
-    ImportFormat,
-    ImportOptions,
-    ImportSummary,
-    ImportValidationResult,
-    JobMetadata,
-    JobProgress,
-    JobResult,
-    JobStatus,
-    JobType,
-)
-
-logger = logging.getLogger(__name__)
-
-# Initialize Celery app
-settings = get_config()
-celery_app = Celery(
-    "memory_export_import",
-    broker=settings.celery_broker_url,  # ADR-INF-5: Explicit Celery DB
-    backend=settings.celery_broker_url,  # ADR-INF-5: Explicit Celery DB
-)
-
-# Redis client for job tracking
-redis_client = redis.from_url(settings.redis_url)
-
-# Configuration
-CELERY_CONFIG = {
-    "task_serializer": "json",
-    "accept_content": ["json"],
-    "result_serializer": "json",
-    "timezone": "UTC",
-    "enable_utc": True,
-    "task_track_started": True,
-    "task_acks_late": True,
-    "task_time_limit": 3600,  # 1 hour
-    "task_soft_time_limit": 3000,  # 50 minutes
-    "worker_prefetch_multiplier": 1,
-    "worker_max_tasks_per_child": 1000,
-}
-
-celery_app.conf.update(CELERY_CONFIG)
-
-
-class MemoryExportImportService:
-    """Service for handling memory export and import operations."""
-
-    def __init__(self, db: Session):
-        self.db = db
-        self.settings = get_config()
-        self._loop = None
-
-    async def create_export_job(
-        self,
-        user_id: uuid.UUID,
-        export_request: ExportRequest,
-        metadata: JobMetadata | None = None,
-    ) -> uuid.UUID:
-        """Create and queue an export job."""
-
-        # Generate job ID
-        job_id = uuid.uuid4()
-
-        # Initialize job status in Redis
-        job_data = {
-            "job_id": str(job_id),
-            "job_type": JobType.EXPORT,
-            "status": JobStatus.PENDING,
-            "user_id": str(user_id),
-            "request_data": export_request.model_dump(),
-            "metadata": metadata.model_dump() if metadata else {},
-            "created_at": datetime.now(UTC).isoformat(),
-            "progress": JobProgress(total=0, processed=0, skipped=0, failed=0).model_dump(),
-        }
-
-        # Store in Redis with TTL
-        await self._store_job_status(job_id, job_data)
-
-        # Queue the task
-        export_task.delay(
-            job_id=str(job_id),
-            user_id=str(user_id),
-            request_data=export_request.model_dump(),
-            metadata=metadata.model_dump() if metadata else {},
-        )
-
-        logger.info(f"Created export job {job_id} for user {user_id}")
-        return job_id
-
-    async def create_import_job(
-        self,
-        user_id: uuid.UUID,
-        import_file: UploadFile,
-        import_options: ImportOptions,
-        metadata: JobMetadata | None = None,
-    ) -> uuid.UUID:
-        """Create and queue an import job."""
-
-        # Validate file upload
-        file_validation = await self._validate_upload_file(import_file)
-        if not file_validation["is_valid"]:
-            raise HTTPException(status_code=400, detail=file_validation["error"])
-
-        # Generate job ID
-        job_id = uuid.uuid4()
-
-        # Save uploaded file temporarily
-        temp_path = await self._save_uploaded_file(import_file, job_id)
-
-        # Initialize job status in Redis
-        job_data = {
-            "job_id": str(job_id),
-            "job_type": JobType.IMPORT,
-            "status": JobStatus.PENDING,
-            "user_id": str(user_id),
-            "request_data": {
-                "import_options": import_options.model_dump(),
-                "file_info": file_validation["file_info"],
-            },
-            "metadata": metadata.model_dump() if metadata else {},
-            "created_at": datetime.now(UTC).isoformat(),
-            "temp_file_path": temp_path,
-            "progress": JobProgress(total=0, processed=0, skipped=0, failed=0).model_dump(),
-        }
-
-        # Store in Redis
-        await self._store_job_status(job_id, job_data)
-
-        # Queue the task
-        import_task.delay(
-            job_id=str(job_id),
-            user_id=str(user_id),
-            temp_file_path=temp_path,
-            import_options=import_options.model_dump(),
-            metadata=metadata.model_dump() if metadata else {},
-        )
-
-        logger.info(f"Created import job {job_id} for user {user_id}")
-        return job_id
-
-    async def get_job_status(self, job_id: uuid.UUID) -> dict[str, Any] | None:
-        """Get the current status of a job."""
-
-        # Try to get from Redis first (for live jobs)
-        job_data = await self._get_job_status_from_redis(job_id)
-
-        if job_data:
-            return job_data
-
-        # If not in Redis, try to get from database (for completed jobs)
-        return await self._get_job_status_from_db(job_id)
-
-    async def cancel_job(self, job_id: uuid.UUID, user_id: uuid.UUID) -> bool:
-        """Cancel a running job."""
-
-        # Get job status
-        job_data = await self.get_job_status(job_id)
-        if not job_data:
-            return False
-
-        # Check ownership
-        if job_data.get("user_id") != str(user_id):
-            raise HTTPException(status_code=403, detail="Not authorized to cancel this job")
-
-        # Check if job can be cancelled
-        if job_data["status"] not in [JobStatus.PENDING, JobStatus.PROCESSING]:
-            return False
-
-        # Update status
-        job_data["status"] = JobStatus.CANCELLED
-        job_data["finished_at"] = datetime.now(UTC).isoformat()
-
-        await self._store_job_status(job_id, job_data)
-
-        # Revoke the Celery task
-        celery_app.control.revoke(str(job_id), terminate=True)
-
-        logger.info(f"Cancelled job {job_id}")
-        return True
-
-    async def list_user_jobs(
-        self,
-        user_id: uuid.UUID,
-        job_type: JobType | None = None,
-        status: JobStatus | None = None,
-        limit: int = 20,
-        offset: int = 0,
-    ) -> list[dict[str, Any]]:
-        """List jobs for a user with optional filters."""
-
-        # Get job IDs from Redis sorted set
-        job_ids = await self._get_user_job_ids(user_id)
-
-        jobs = []
-        for job_id in job_ids[offset : offset + limit]:
-            job_data = await self.get_job_status(uuid.UUID(job_id))
-            if job_data:
-                # Apply filters
-                if job_type and job_data.get("job_type") != job_type:
-                    continue
-                if status and job_data.get("status") != status:
-                    continue
-
-                jobs.append(job_data)
-
-        return jobs
-
-    # Private helper methods
-
-    async def _store_job_status(self, job_id: uuid.UUID, job_data: dict[str, Any]) -> None:
-        """Store job status in Redis."""
-
-        redis_key = f"job:{job_id}"
-        await asyncio.get_event_loop().run_in_executor(
-            None,
-            lambda: redis_client.setex(
-                redis_key,
-                timedelta(hours=48),  # 48 hour TTL
-                json.dumps(job_data),
-            ),
-        )
-
-        # Add to user's job list
-        user_id = job_data["user_id"]
-        user_jobs_key = f"user_jobs:{user_id}"
-        await asyncio.get_event_loop().run_in_executor(
-            None,
-            lambda: redis_client.zadd(user_jobs_key, {str(job_id): datetime.now(UTC).timestamp()}),
-        )
-
-        await self._persist_job_history(job_data)
-
-    async def _get_job_status_from_redis(self, job_id: uuid.UUID) -> dict[str, Any] | None:
-        """Get job status from Redis."""
-
-        redis_key = f"job:{job_id}"
-        data = await asyncio.get_event_loop().run_in_executor(
-            None, lambda: redis_client.get(redis_key)
-        )
-
-        if data:
-            return json.loads(data)
-        return None
-
-    async def _get_job_status_from_db(self, job_id: uuid.UUID) -> dict[str, Any] | None:
-        """Get job status from PostgreSQL database."""
-
-        if not self.db:
-            return None
-
-        def fetch() -> ExportImportJob | None:
-            return self.db.get(ExportImportJob, job_id)
-
-        job = await asyncio.get_event_loop().run_in_executor(None, fetch)
-        if not job:
-            return None
-
-        def _dt(value: datetime | None) -> str | None:
-            return value.isoformat() if value else None
-
-        payload = {
-            "job_id": str(job.job_id),
-            "job_type": job.job_type,
-            "status": job.status,
-            "user_id": str(job.user_id),
-            "request_data": job.request_data or {},
-            "result": job.result_data or {},
-            "metadata": job.job_metadata or {},
-            "file_path": job.file_path,
-            "error_message": job.error_message,
-            "created_at": _dt(job.created_at),
-            "started_at": _dt(job.started_at),
-            "updated_at": _dt(
-                job.finished_at
-            ),  # ExportImportJob has no updated_at; use finished_at
-            "finished_at": _dt(job.finished_at),
-        }
-        return payload
-
-    async def _get_user_job_ids(self, user_id: uuid.UUID) -> list[str]:
-        """Get job IDs for a user from Redis sorted set."""
-
-        user_jobs_key = f"user_jobs:{user_id}"
-        job_ids = await asyncio.get_event_loop().run_in_executor(
-            None,
-            lambda: redis_client.zrevrange(user_jobs_key, 0, -1),  # Get all, newest first
-        )
-        if job_ids:
-            return [job_id.decode() if isinstance(job_id, bytes) else job_id for job_id in job_ids]
-
-        if not self.db:
-            return []
-
-        def fetch() -> list[str]:
-            # SQLAlchemy 2.0: use select() instead of query() for AsyncSession compatibility
-            stmt = (
-                select(ExportImportJob.job_id)
-                .where(ExportImportJob.user_id == user_id)
-                .order_by(ExportImportJob.created_at.desc())
-                .limit(50)
-            )
-            result = self.db.execute(stmt)
-            return [str(row[0]) for row in result.scalars().all()]
-
-        return await asyncio.get_event_loop().run_in_executor(None, fetch)
-
-    @staticmethod
-    def _parse_datetime(value: str | None) -> datetime | None:
-        if not value:
-            return None
-        try:
-            return datetime.fromisoformat(value.replace("Z", "+00:00"))
-        except ValueError:
-            return None
-
-    async def _persist_job_history(self, job_data: dict[str, Any]) -> None:
-        if not self.db:
-            return
-
-        def upsert() -> None:
-            session = self.db
-            job_uuid = uuid.UUID(job_data["job_id"])
-            job = session.get(ExportImportJob, job_uuid)
-            if not job:
-                job = ExportImportJob(
-                    job_id=job_uuid,
-                    user_id=str(job_data["user_id"]),
-                    job_type=job_data.get("job_type", JobType.EXPORT),
-                    request_data=job_data.get("request_data") or {},
-                    job_metadata=job_data.get("metadata") or {},
-                    created_at=self._parse_datetime(job_data.get("created_at"))
-                    or datetime.now(UTC),
-                )
-                session.add(job)
-
-            job.status = job_data.get("status", job.status)
-            job.result_data = job_data.get("result") or job.result_data or {}
-            metadata = job_data.get("metadata")
-            if metadata:
-                job.job_metadata = metadata
-            job.file_path = job_data.get("result", {}).get("file_path") or job.file_path
-            job.error_message = job_data.get("error_message")
-            job.started_at = self._parse_datetime(job_data.get("started_at")) or job.started_at
-            job.finished_at = self._parse_datetime(job_data.get("finished_at")) or job.finished_at
-            if not job.request_data:
-                job.request_data = job_data.get("request_data") or {}
-            session.commit()
-
-        await asyncio.get_event_loop().run_in_executor(None, upsert)
-
-    async def _validate_upload_file(self, file: UploadFile) -> dict[str, Any]:
-        """Validate uploaded file for import."""
-
-        # Check file size
-        if file.size and file.size > self.settings.max_import_file_size_mb * 1024 * 1024:
-            return {
-                "is_valid": False,
-                "error": f"File size exceeds {self.settings.max_import_file_size_mb}MB limit",
-            }
-
-        # Check file extension
-        allowed_extensions = [".json", ".jsonl", ".csv"]
-        if not any(file.filename.lower().endswith(ext) for ext in allowed_extensions):
-            return {
-                "is_valid": False,
-                "error": f"File type not supported. Allowed types: {', '.join(allowed_extensions)}",
-            }
-
-        # Determine file format
-        if file.filename.lower().endswith(".json"):
-            file_format = ImportFormat.JSON
-        elif file.filename.lower().endswith(".jsonl"):
-            file_format = ImportFormat.JSONL
-        elif file.filename.lower().endswith(".csv"):
-            file_format = ImportFormat.CSV
-        else:
-            file_format = None
-
-        return {
-            "is_valid": True,
-            "file_format": file_format,
-            "file_info": {
-                "filename": file.filename,
-                "size": file.size,
-                "content_type": file.content_type,
-                "format": file_format,
-            },
-        }
-
-    async def _save_uploaded_file(self, file: UploadFile, job_id: uuid.UUID) -> str:
-        """Save uploaded file to temporary storage."""
-
-        # Create temp directory if it doesn't exist
-        temp_dir = Path("/tmp/imports")
-        temp_dir.mkdir(exist_ok=True)
-
-        # Generate unique filename
-        file_extension = Path(file.filename).suffix
-        temp_filename = f"{job_id}{file_extension}"
-        temp_path = temp_dir / temp_filename
-
-        # Save file
-        content = await file.read()
-        with open(temp_path, "wb") as f:
-            f.write(content)
-
-        # Reset file pointer
-        await file.seek(0)
-
-        return str(temp_path)
-
-
-# Celery tasks for background processing
-
-
-@celery_app.task(bind=True)
-def export_task(
-    self,
-    job_id: str,
-    user_id: str,
-    request_data: dict[str, Any],
-    _metadata: dict[str, Any],
-) -> dict[str, Any]:
-    """Background task for exporting memories."""
-
-    try:
-        # Initialize database session
-        db = next(get_db())
-        service = MemoryExportImportService(db)
-
-        # Update job status to processing
-        job_data = asyncio.run(service._get_job_status_from_redis(uuid.UUID(job_id)))
-        job_data["status"] = JobStatus.PROCESSING
-        job_data["started_at"] = datetime.now(UTC).isoformat()
-        asyncio.run(service._store_job_status(uuid.UUID(job_id), job_data))
-
-        # Parse request data
-        export_request = ExportRequest(**request_data)
-
-        # Process export
-        result = asyncio.run(_process_export(db, job_id, user_id, export_request, self))
-
-        return result
-
-    except Exception as e:
-        logger.exception(f"Export task {job_id} failed: {str(e)}")
-
-        # Update job status to failed
-        job_data = asyncio.run(service._get_job_status_from_redis(uuid.UUID(job_id)))
-        job_data["status"] = JobStatus.FAILED
-        job_data["error_message"] = str(e)
-        job_data["finished_at"] = datetime.now(UTC).isoformat()
-        asyncio.run(service._store_job_status(uuid.UUID(job_id), job_data))
-
-        return {"success": False, "error": str(e)}
-
-    finally:
-        db.close()
-
-
-@celery_app.task(bind=True)
-def import_task(
-    self,
-    job_id: str,
-    user_id: str,
-    temp_file_path: str,
-    import_options: dict[str, Any],
-    _metadata: dict[str, Any],
-) -> dict[str, Any]:
-    """Background task for importing memories."""
-
-    try:
-        # Initialize database session
-        db = next(get_db())
-        service = MemoryExportImportService(db)
-
-        # Update job status to processing
-        job_data = asyncio.run(service._get_job_status_from_redis(uuid.UUID(job_id)))
-        job_data["status"] = JobStatus.PROCESSING
-        job_data["started_at"] = datetime.now(UTC).isoformat()
-        asyncio.run(service._store_job_status(uuid.UUID(job_id), job_data))
-
-        # Parse import options
-        options = ImportOptions(**import_options)
-
-        # Process import
-        result = asyncio.run(_process_import(db, job_id, user_id, temp_file_path, options, self))
-
-        # Cleanup temporary file
-        os.unlink(temp_file_path)
-
-        return result
-
-    except Exception as e:
-        logger.exception(f"Import task {job_id} failed: {str(e)}")
-
-        # Update job status to failed
-        job_data = asyncio.run(service._get_job_status_from_redis(uuid.UUID(job_id)))
-        job_data["status"] = JobStatus.FAILED
-        job_data["error_message"] = str(e)
-        job_data["finished_at"] = datetime.now(UTC).isoformat()
-        asyncio.run(service._store_job_status(uuid.UUID(job_id), job_data))
-
-        # Cleanup temporary file
-        with contextlib.suppress(BaseException):
-            os.unlink(temp_file_path)
-
-        return {"success": False, "error": str(e)}
-
-    finally:
-        db.close()
-
-
-async def _process_export(
-    db: Session, job_id: str, user_id: str, export_request: ExportRequest, task
-) -> dict[str, Any]:
-    """Process memory export and generate file."""
-
-    service = MemoryExportImportService(db)
-
-    # SQLAlchemy 2.0: Build select statement dynamically based on filters
-    stmt = select(Memory).where(Memory.user_id == uuid.UUID(user_id))
-
-    if export_request.filters:
-        filters = export_request.filters
-
-        if filters.memory_ids:
-            stmt = stmt.where(Memory.memory_id.in_(filters.memory_ids))
-
-        if filters.category_ids:
-            # Assuming memory has category relationship
-            # stmt = stmt.where(Memory.category_id.in_(filters.category_ids))
-            pass
-
-        if filters.date_from:
-            stmt = stmt.where(Memory.created_at >= filters.date_from)
-
-        if filters.date_to:
-            stmt = stmt.where(Memory.created_at <= filters.date_to)
-
-        if filters.importance_levels:
-            stmt = stmt.where(Memory.importance.in_(filters.importance_levels))
-
-        if not filters.include_deleted:
-            stmt = stmt.where(Memory.deleted_at.is_(None))
-
-        if filters.team_id:
-            # Query memories shared with team
-            # This would require joining with permissions table
-            pass
-
-    # Get total count for progress tracking
-    total_count = db.execute(select(func.count()).select_from(stmt.subquery())).scalar()
-
-    if total_count == 0:
-        raise ValueError("No memories found matching the specified filters")
-
-    # Update progress
-    progress = JobProgress(total=total_count, processed=0, skipped=0, failed=0)
-    await _update_job_progress(job_id, progress)
-
-    # Generate file based on format
-    start_time = datetime.now(UTC)
-
-    if export_request.format == ExportFormat.JSON:
-        file_path, file_size = await _export_json(db, stmt, export_request, job_id, task)
-    elif export_request.format == ExportFormat.JSONL:
-        file_path, file_size = await _export_jsonl(db, stmt, export_request, job_id, task)
-    elif export_request.format == ExportFormat.CSV:
-        file_path, file_size = await _export_csv(db, stmt, export_request, job_id, task)
-    elif export_request.format == ExportFormat.MARKDOWN:
-        file_path, file_size = await _export_markdown(db, stmt, export_request, job_id, task)
-    elif export_request.format == ExportFormat.EXCEL:
-        file_path, file_size = await _export_excel(db, stmt, export_request, job_id, task)
-    else:
-        raise ValueError(f"Unsupported export format: {export_request.format}")
-
-    duration = (datetime.now(UTC) - start_time).total_seconds()
-
-    # Compress file if requested
-    if export_request.compress:
-        file_path, file_size = await _compress_file(file_path)
-
-    # Generate download URL (this would depend on your file storage setup)
-    download_url = f"/api/v1/exports/download/{job_id}/{Path(file_path).name}"
-
-    # Create job result
-    result = JobResult(
-        file_name=Path(file_path).name,
-        file_path=file_path,
-        file_size_bytes=file_size,
-        download_url=download_url,
-        mime_type=mimetypes.guess_type(file_path)[0],
-        expires_at=datetime.now(UTC) + timedelta(hours=export_request.expires_in_hours),
-        record_count=total_count,
-        success_count=total_count,
-        error_count=0,
-    )
-
-    # Update job status to completed
-    job_data = await service._get_job_status_from_redis(uuid.UUID(job_id))
-    job_data["status"] = JobStatus.COMPLETED
-    job_data["progress"] = progress.model_dump()
-    job_data["result"] = result.model_dump()
-    job_data["finished_at"] = datetime.now(UTC).isoformat()
-
-    await service._store_job_status(uuid.UUID(job_id), job_data)
-
-    return {
-        "success": True,
-        "result": result.model_dump(),
-        "summary": ExportSummary(
-            total_memories=total_count,
-            exported_memories=total_count,
-            filters_applied=export_request.filters is not None,
-            format=export_request.format,
-            file_size_bytes=file_size,
-            compressed=export_request.compress,
-            duration_seconds=duration,
-            includes_metadata=export_request.include_metadata,
-        ).model_dump(),
-    }
-
-
-async def _process_import(
-    db: Session,
-    job_id: str,
-    user_id: str,
-    temp_file_path: str,
-    import_options: ImportOptions,
-    _task,
-) -> dict[str, Any]:
-    """Process memory import from file."""
-
-    service = MemoryExportImportService(db)
-    start_time = datetime.now(UTC)
-
-    # Validate and parse import file
-    validation_result = await _validate_import_file(temp_file_path)
-    if not validation_result.is_valid:
-        raise ValueError(f"Invalid import file: {', '.join(validation_result.validation_errors)}")
-
-    # Get total records for progress tracking
-    total_records = validation_result.total_records
-
-    if total_records == 0:
-        raise ValueError("No valid records found in import file")
-
-    # Update progress
-    progress = JobProgress(total=total_records, processed=0, skipped=0, failed=0)
-    await _update_job_progress(job_id, progress)
-
-    # Process records based on format
-    records = await _read_import_file(temp_file_path, validation_result.format)
-
-    successful_imports = 0
-    updates = 0
-    skipped_records = 0
-    failed_records = 0
-    errors = []
-    new_entities_created = {}
-
-    for i, record_data in enumerate(records):
-        try:
-            # Update progress
-            progress.processed = i + 1
-            progress.current_item = f"Processing record {i + 1} of {total_records}"
-            await _update_job_progress(job_id, progress)
-
-            # Process individual record
-            result = await _process_import_record(
-                db, record_data, uuid.UUID(user_id), import_options
-            )
-
-            if result["action"] == "created":
-                successful_imports += 1
-            elif result["action"] == "updated":
-                updates += 1
-            elif result["action"] == "skipped":
-                skipped_records += 1
-            elif result["action"] == "failed":
-                failed_records += 1
-                if result.get("error"):
-                    errors.append(f"Record {i + 1}: {result['error']}")
-
-            # Track new entities created
-            if result.get("new_entities"):
-                for entity_type, count in result["new_entities"].items():
-                    new_entities_created[entity_type] = (
-                        new_entities_created.get(entity_type, 0) + count
-                    )
-
-        except Exception as e:
-            failed_records += 1
-            errors.append(f"Record {i + 1}: {str(e)}")
-            logger.exception(f"Error processing import record {i + 1}: {str(e)}")
-
-    duration = (datetime.now(UTC) - start_time).total_seconds()
-
-    # Create import summary
-    summary = ImportSummary(
-        total_records=total_records,
-        processed_records=progress.processed,
-        successful_imports=successful_imports,
-        updates=updates,
-        skipped_records=skipped_records,
-        failed_records=failed_records,
-        new_entities_created=new_entities_created,
-        duration_seconds=duration,
-        errors=errors[:10],  # Limit to first 10 errors
-        warnings=[],
-    )
-
-    # Update job status to completed
-    job_data = await service._get_job_status_from_redis(uuid.UUID(job_id))
-    job_data["status"] = JobStatus.COMPLETED
-    job_data["progress"] = progress.model_dump()
-    job_data["result"] = summary.model_dump()
-    job_data["finished_at"] = datetime.now(UTC).isoformat()
-
-    await service._store_job_status(uuid.UUID(job_id), job_data)
-
-    return {"success": True, "summary": summary.model_dump()}
-
-
-# Export format handlers
-
-
-async def _export_json(
-    db: Session, stmt, export_request: ExportRequest, job_id: str, _task
-) -> tuple[str, int]:
-    """Export memories as JSON file."""
-
-    memories = []
-    output_dir = Path("/tmp/exports")
-    output_dir.mkdir(exist_ok=True)
-    output_file = output_dir / f"{job_id}.json"
-
-    # Process in batches for memory efficiency
-    batch_size = 1000
-    offset = 0
-
-    while True:
-        # SQLAlchemy 2.0: Use execute with offset/limit on the statement
-        batch = db.execute(stmt.offset(offset).limit(batch_size)).scalars().all()
-        if not batch:
-            break
-
-        for memory in batch:
-            memory_data = {
-                "id": str(memory.memory_id),
-                "title": memory.title,
-                "content": memory.content,
-                "created_at": memory.created_at.isoformat() if memory.created_at else None,
-                "updated_at": memory.updated_at.isoformat() if memory.updated_at else None,
-                "importance": memory.importance,
-                "tags": memory.tags if hasattr(memory, "tags") else [],
-            }
-
-            if export_request.include_metadata:
-                memory_data.update(
-                    {
-                        "user_id": str(memory.user_id),
-                        "embedding_id": str(memory.embedding_id) if memory.embedding_id else None,
-                        "metadata": memory.metadata if hasattr(memory, "metadata") else {},
-                    }
-                )
-
-            memories.append(memory_data)
-
-            # Update progress
-            progress_data = await _get_job_progress(job_id)
-            if progress_data:
-                progress_data["processed"] += 1
-                await _update_job_progress(job_id, JobProgress(**progress_data))
-
-        offset += batch_size
-
-    # Write JSON file
-    with open(output_file, "w", encoding="utf-8") as f:
-        json.dump(memories, f, indent=2, ensure_ascii=False)
-
-    file_size = output_file.stat().st_size
-    return str(output_file), file_size
-
-
-async def _export_jsonl(
-    db: Session, stmt, export_request: ExportRequest, job_id: str, _task
-) -> tuple[str, int]:
-    """Export memories as JSONL file."""
-
-    output_dir = Path("/tmp/exports")
-    output_dir.mkdir(exist_ok=True)
-    output_file = output_dir / f"{job_id}.jsonl"
-
-    processed = 0
-
-    with open(output_file, "w", encoding="utf-8") as f:
-        # Process in batches
-        batch_size = 1000
-        offset = 0
-
-        while True:
-            # SQLAlchemy 2.0: Use execute with offset/limit on the statement
-            batch = db.execute(stmt.offset(offset).limit(batch_size)).scalars().all()
-            if not batch:
-                break
-
-            for memory in batch:
-                memory_data = {
-                    "id": str(memory.memory_id),
-                    "title": memory.title,
-                    "content": memory.content,
-                    "created_at": memory.created_at.isoformat() if memory.created_at else None,
-                    "updated_at": memory.updated_at.isoformat() if memory.updated_at else None,
-                    "importance": memory.importance,
-                }
-
-                if export_request.include_metadata:
-                    memory_data.update(
-                        {
-                            "user_id": str(memory.user_id),
-                            "embedding_id": str(memory.embedding_id)
-                            if memory.embedding_id
-                            else None,
-                            "metadata": memory.metadata if hasattr(memory, "metadata") else {},
-                        }
-                    )
-
-                f.write(json.dumps(memory_data, ensure_ascii=False) + "\n")
-                processed += 1
-
-            offset += batch_size
-
-            # Update progress (without count for efficiency)
-            progress = JobProgress(total=0, processed=processed, skipped=0, failed=0)
-            await _update_job_progress(job_id, progress)
-
-    file_size = output_file.stat().st_size
-    return str(output_file), file_size
-
-
-async def _export_csv(
-    db: Session, stmt, export_request: ExportRequest, job_id: str, _task
-) -> tuple[str, int]:
-    """Export memories as CSV file."""
-
-    output_dir = Path("/tmp/exports")
-    output_dir.mkdir(exist_ok=True)
-    output_file = output_dir / f"{job_id}.csv"
-
-    processed = 0
-
-    with open(output_file, "w", newline="", encoding="utf-8") as f:
-        writer = csv.writer(f)
-
-        # Write header
-        headers = ["id", "title", "content", "created_at", "updated_at", "importance"]
-        if export_request.include_metadata:
-            headers.extend(["user_id", "embedding_id", "tags", "metadata"])
-
-        writer.writerow(headers)
-
-        # Process in batches
-        batch_size = 1000
-        offset = 0
-
-        while True:
-            # SQLAlchemy 2.0: Use execute with offset/limit on the statement
-            batch = db.execute(stmt.offset(offset).limit(batch_size)).scalars().all()
-            if not batch:
-                break
-
-            for memory in batch:
-                row = [
-                    str(memory.memory_id),
-                    memory.title or "",
-                    memory.content or "",
-                    memory.created_at.isoformat() if memory.created_at else "",
-                    memory.updated_at.isoformat() if memory.updated_at else "",
-                    memory.importance or "",
-                ]
-
-                if export_request.include_metadata:
-                    row.extend(
-                        [
-                            str(memory.user_id),
-                            str(memory.embedding_id) if memory.embedding_id else "",
-                            json.dumps(memory.tags) if hasattr(memory, "tags") else "[]",
-                            json.dumps(memory.metadata) if hasattr(memory, "metadata") else "{}",
-                        ]
-                    )
-
-                writer.writerow(row)
-                processed += 1
-
-            offset += batch_size
-
-            # Update progress (without count for efficiency)
-            progress = JobProgress(total=0, processed=processed, skipped=0, failed=0)
-            await _update_job_progress(job_id, progress)
-
-    file_size = output_file.stat().st_size
-    return str(output_file), file_size
-
-
-async def _export_markdown(
-    db: Session, stmt, export_request: ExportRequest, job_id: str, _task
-) -> tuple[str, int]:
-    """Export memories as Markdown file."""
-
-    output_dir = Path("/tmp/exports")
-    output_dir.mkdir(exist_ok=True)
-    output_file = output_dir / f"{job_id}.md"
-
-    processed = 0
-
-    with open(output_file, "w", encoding="utf-8") as f:
-        f.write("# Memory Export\n\n")
-        f.write(f"Exported on: {datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S UTC')}\n\n")
-
-        if export_request.filters:
-            f.write("## Filters Applied\n\n")
-            # Add filter details here
-            f.write("\n")
-
-        f.write("---\n\n")
-
-        # Process in batches
-        batch_size = 1000
-        offset = 0
-
-        while True:
-            # SQLAlchemy 2.0: Use execute with offset/limit on the statement
-            batch = db.execute(stmt.offset(offset).limit(batch_size)).scalars().all()
-            if not batch:
-                break
-
-            for memory in batch:
-                f.write(f"# {memory.title or 'Untitled'}\n\n")
-
-                if export_request.include_metadata:
-                    f.write(f"**ID:** `{memory.memory_id}`\n\n")
-                    f.write(
-                        f"**Created:** {memory.created_at.strftime('%Y-%m-%d %H:%M:%S UTC') if memory.created_at else 'Unknown'}\n\n"
-                    )
-                    f.write(
-                        f"**Updated:** {memory.updated_at.strftime('%Y-%m-%d %H:%M:%S UTC') if memory.updated_at else 'Unknown'}\n\n"
-                    )
-                    f.write(f"**Importance:** {memory.importance or 'Not set'}\n\n")
-
-                if hasattr(memory, "tags") and memory.tags:
-                    f.write(f"**Tags:** {', '.join(memory.tags)}\n\n")
-
-                f.write(f"{memory.content or 'No content'}\n\n")
-                f.write("---\n\n")
-
-                processed += 1
-
-            offset += batch_size
-
-            # Update progress (without count for efficiency)
-            progress = JobProgress(total=0, processed=processed, skipped=0, failed=0)
-            await _update_job_progress(job_id, progress)
-
-    file_size = output_file.stat().st_size
-    return str(output_file), file_size
-
-
-async def _export_excel(
-    db: Session, stmt, export_request: ExportRequest, job_id: str, _task
-) -> tuple[str, int]:
-    """Export memories as Excel file."""
-
-    output_dir = Path("/tmp/exports")
-    output_dir.mkdir(exist_ok=True)
-    output_file = output_dir / f"{job_id}.xlsx"
-
-    # Collect all data first (more efficient for Excel)
-    memories_data = []
-
-    batch_size = 1000
-    offset = 0
-    processed = 0
-
-    while True:
-        # SQLAlchemy 2.0: Use execute with offset/limit on the statement
-        batch = db.execute(stmt.offset(offset).limit(batch_size)).scalars().all()
-        if not batch:
-            break
-
-        for memory in batch:
-            memory_data = {
-                "id": str(memory.memory_id),
-                "title": memory.title or "",
-                "content": memory.content or "",
-                "created_at": memory.created_at.isoformat() if memory.created_at else "",
-                "updated_at": memory.updated_at.isoformat() if memory.updated_at else "",
-                "importance": memory.importance or "",
-            }
-
-            if export_request.include_metadata:
-                memory_data.update(
-                    {
-                        "user_id": str(memory.user_id),
-                        "embedding_id": str(memory.embedding_id) if memory.embedding_id else "",
-                        "tags": json.dumps(memory.tags) if hasattr(memory, "tags") else "[]",
-                        "metadata": json.dumps(memory.metadata)
-                        if hasattr(memory, "metadata")
-                        else "{}",
-                    }
-                )
-
-            memories_data.append(memory_data)
-            processed += 1
-
-        offset += batch_size
-
-        # Update progress (without count for efficiency)
-        progress = JobProgress(total=0, processed=processed, skipped=0, failed=0)
-        await _update_job_progress(job_id, progress)
-
-    # Create Excel file
-    df = pd.DataFrame(memories_data)
-
-    with pd.ExcelWriter(output_file, engine="xlsxwriter") as writer:
-        df.to_excel(writer, sheet_name="Memories", index=False)
-
-        # Adjust column widths
-        worksheet = writer.sheets["Memories"]
-        for idx, col in enumerate(df.columns):
-            max_length = max(df[col].astype(str).map(len).max(), len(col))
-            worksheet.set_column(idx, idx, min(max_length + 2, 50))
-
-    file_size = output_file.stat().st_size
-    return str(output_file), file_size
-
-
-# Import helpers
-
-
-async def _validate_import_file(file_path: str) -> ImportValidationResult:
-    """Validate import file format and structure."""
-
-    file_path = Path(file_path)
-
-    if file_path.suffix.lower() == ".json":
-        return await _validate_json_import(file_path)
-    elif file_path.suffix.lower() == ".jsonl":
-        return await _validate_jsonl_import(file_path)
-    elif file_path.suffix.lower() == ".csv":
-        return await _validate_csv_import(file_path)
-    else:
-        return ImportValidationResult(is_valid=False, validation_errors=["Unsupported file format"])
-
-
-async def _validate_json_import(file_path: Path) -> ImportValidationResult:
-    """Validate JSON import file."""
-
-    try:
-        with open(file_path, encoding="utf-8") as f:
-            data = json.load(f)
-
-        if not isinstance(data, list):
-            return ImportValidationResult(
-                is_valid=False,
-                validation_errors=["JSON file must contain an array of memory objects"],
-            )
-
-        # Sample first few records for validation
-        sample_records = data[:10]
-        validation_errors = []
-
-        for i, record in enumerate(sample_records):
-            if not isinstance(record, dict):
-                validation_errors.append(f"Record {i + 1} is not an object")
-                continue
-
-            # Check required fields
-            if not record.get("title") and not record.get("content"):
-                validation_errors.append(f"Record {i + 1} must have either title or content")
-
-        return ImportValidationResult(
-            is_valid=len(validation_errors) == 0,
-            format=ImportFormat.JSON,
-            total_records=len(data),
-            sample_records=sample_records,
-            validation_errors=validation_errors,
-        )
-
-    except json.JSONDecodeError as e:
-        return ImportValidationResult(
-            is_valid=False, validation_errors=[f"Invalid JSON format: {str(e)}"]
-        )
-    except Exception as e:
-        return ImportValidationResult(
-            is_valid=False, validation_errors=[f"Error reading file: {str(e)}"]
-        )
-
-
-async def _validate_jsonl_import(file_path: Path) -> ImportValidationResult:
-    """Validate JSONL import file."""
-
-    try:
-        records = []
-        validation_errors = []
-        line_number = 0
-
-        with open(file_path, encoding="utf-8") as f:
-            for line in f:
-                line_number += 1
-                line = line.strip()
-                if not line:
-                    continue
-
-                try:
-                    record = json.loads(line)
-                    if not isinstance(record, dict):
-                        validation_errors.append(f"Line {line_number} is not a valid object")
-                        continue
-
-                    records.append(record)
-
-                    # Sample first 10 records for validation
-                    if len(records) <= 10 and not record.get("title") and not record.get("content"):
-                            validation_errors.append(
-                                f"Line {line_number} must have either title or content"
-                            )
-
-                except json.JSONDecodeError as e:
-                    validation_errors.append(f"Line {line_number} has invalid JSON: {str(e)}")
-
-        return ImportValidationResult(
-            is_valid=len(validation_errors) == 0,
-            format=ImportFormat.JSONL,
-            total_records=len(records),
-            sample_records=records[:10],
-            validation_errors=validation_errors,
-        )
-
-    except Exception as e:
-        return ImportValidationResult(
-            is_valid=False, validation_errors=[f"Error reading file: {str(e)}"]
-        )
-
-
-async def _validate_csv_import(file_path: Path) -> ImportValidationResult:
-    """Validate CSV import file."""
-
-    try:
-        records = []
-        validation_errors = []
-
-        with open(file_path, encoding="utf-8") as f:
-            reader = csv.DictReader(f)
-
-            # Check required columns
-            required_columns = ["title", "content"]
-            missing_columns = [col for col in required_columns if col not in reader.fieldnames]
-
-            if missing_columns:
-                validation_errors.append(f"Missing required columns: {', '.join(missing_columns)}")
-
-            # Sample first 10 records
-            for i, row in enumerate(reader):
-                if i >= 10:
-                    break
-
-                if not row.get("title") and not row.get("content"):
-                    validation_errors.append(f"Row {i + 2} must have either title or content")
-
-                records.append(dict(row))
-
-        # Count total records
-        total_records = sum(1 for _ in open(file_path, encoding="utf-8")) - 1  # Subtract header
-
-        return ImportValidationResult(
-            is_valid=len(validation_errors) == 0,
-            format=ImportFormat.CSV,
-            total_records=total_records,
-            sample_records=records,
-            validation_errors=validation_errors,
-        )
-
-    except Exception as e:
-        return ImportValidationResult(
-            is_valid=False, validation_errors=[f"Error reading CSV file: {str(e)}"]
-        )
-
-
-async def _read_import_file(file_path: str, format: ImportFormat) -> list[dict[str, Any]]:
-    """Read records from import file."""
-
-    file_path = Path(file_path)
-
-    if format == ImportFormat.JSON:
-        with open(file_path, encoding="utf-8") as f:
-            return json.load(f)
-
-    elif format == ImportFormat.JSONL:
-        records = []
-        with open(file_path, encoding="utf-8") as f:
-            for line in f:
-                line = line.strip()
-                if line:
-                    records.append(json.loads(line))
-        return records
-
-    elif format == ImportFormat.CSV:
-        with open(file_path, encoding="utf-8") as f:
-            reader = csv.DictReader(f)
-            return [dict(row) for row in reader]
-
-
-async def _process_import_record(
-    db: Session,
-    record_data: dict[str, Any],
-    user_id: uuid.UUID,
-    import_options: ImportOptions,
-) -> dict[str, Any]:
-    """Process a single import record."""
-
-    try:
-        # Check for existing memory (by ID or title/content match)
-        existing_memory = None
-
-        if "id" in record_data:
-            try:
-                memory_id = uuid.UUID(record_data["id"])
-                # SQLAlchemy 2.0: Use select() instead of query()
-                stmt = select(Memory).where(
-                    Memory.memory_id == memory_id, Memory.user_id == user_id
-                )
-                existing_memory = db.execute(stmt).scalar_one_or_none()
-            except ValueError:
-                pass  # Invalid UUID, continue with other checks
-
-        if not existing_memory and "title" in record_data:
-            # SQLAlchemy 2.0: Use select() instead of query()
-            stmt = select(Memory).where(
-                Memory.user_id == user_id, Memory.title == record_data["title"]
-            )
-            existing_memory = db.execute(stmt).scalar_one_or_none()
-
-        # Handle conflict based on strategy
-        if existing_memory:
-            if import_options.on_conflict == ImportConflictStrategy.SKIP:
-                return {"action": "skipped", "reason": "existing_memory"}
-            elif import_options.on_conflict == ImportConflictStrategy.FAIL:
-                return {"action": "failed", "error": "Memory already exists"}
-            elif import_options.on_conflict == ImportConflictStrategy.OVERWRITE:
-                if import_options.update_existing:
-                    # Update existing memory
-                    if "title" in record_data:
-                        existing_memory.title = record_data["title"]
-                    if "content" in record_data:
-                        existing_memory.content = record_data["content"]
-                    if "importance" in record_data:
-                        existing_memory.importance = record_data["importance"]
-
-                    existing_memory.updated_at = datetime.now(UTC)
-                    db.commit()
-
-                    return {
-                        "action": "updated",
-                        "memory_id": str(existing_memory.memory_id),
-                    }
-                else:
-                    return {"action": "skipped", "reason": "existing_memory"}
-            elif import_options.on_conflict == ImportConflictStrategy.MERGE:
-                # Merge data intelligently
-                if record_data.get("title") and len(record_data["title"]) > len(
-                    existing_memory.title or ""
-                ):
-                    existing_memory.title = record_data["title"]
-
-                if record_data.get("content"):
-                    if existing_memory.content:
-                        # Append content with separator
-                        existing_memory.content += f"\n\n---\n\n{record_data['content']}"
-                    else:
-                        existing_memory.content = record_data["content"]
-
-                if record_data.get("importance"):
-                    existing_memory.importance = record_data["importance"]
-
-                existing_memory.updated_at = datetime.now(UTC)
-                db.commit()
-
-                return {
-                    "action": "updated",
-                    "memory_id": str(existing_memory.memory_id),
-                }
-
-        # Create new memory
-        if not import_options.dry_run:
-            new_memory = Memory(
-                user_id=user_id,
-                title=record_data.get("title", ""),
-                content=record_data.get("content", ""),
-                importance=record_data.get("importance", "medium"),
-                created_at=datetime.now(UTC),
-                updated_at=datetime.now(UTC),
-            )
-
-            # Handle tags if present
-            if "tags" in record_data and record_data["tags"]:
-                # Assuming tags are stored as JSON array
-                new_memory.tags = record_data["tags"]
-
-            # Handle metadata if present
-            if "metadata" in record_data and record_data["metadata"]:
-                new_memory.metadata = record_data["metadata"]
-
-            db.add(new_memory)
-            db.commit()
-
-            return {"action": "created", "memory_id": str(new_memory.memory_id)}
-        else:
-            return {"action": "validated", "title": record_data.get("title", "")}
-
-    except Exception as e:
-        return {"action": "failed", "error": str(e)}
-
-
-# Utility functions
-
-
-async def _compress_file(file_path: str) -> tuple[str, int]:
-    """Compress file using zip."""
-
-    file_path = Path(file_path)
-    zip_path = file_path.parent / f"{file_path.stem}.zip"
-
-    with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
-        zf.write(file_path, file_path.name)
-
-    # Remove original file
-    file_path.unlink()
-
-    file_size = zip_path.stat().st_size
-    return str(zip_path), file_size
-
-
-async def _update_job_progress(job_id: str, progress: JobProgress) -> None:
-    """Update job progress in Redis."""
-
-    redis_key = f"job:{job_id}"
-    job_data = redis_client.get(redis_key)
-
-    if job_data:
-        job_dict = json.loads(job_data)
-        job_dict["progress"] = progress.model_dump()
-        job_dict["updated_at"] = datetime.now(UTC).isoformat()
-
-        redis_client.setex(redis_key, timedelta(hours=48), json.dumps(job_dict))
-
-
-async def _get_job_progress(job_id: str) -> dict[str, Any] | None:
-    """Get job progress from Redis."""
-
-    redis_key = f"job:{job_id}"
-    job_data = redis_client.get(redis_key)
-
-    if job_data:
-        job_dict = json.loads(job_data)
-        return job_dict.get("progress")
-
-    return None