@datarecce/ui 0.1.29 → 0.1.31

package/recce-source/recce/apis/check_events_api.py

@@ -0,0 +1,353 @@
+"""
+Check Events API endpoints.
+
+This module provides REST endpoints for check events (timeline/conversation),
+proxying requests to Recce Cloud. This feature is only available for cloud users.
+"""
+
+import logging
+from typing import List, Optional
+from uuid import UUID
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from recce.core import default_context
+from recce.event import get_recce_api_token
+from recce.exceptions import RecceException
+from recce.util.cloud.check_events import CheckEventsCloud
+from recce.util.recce_cloud import RecceCloud, RecceCloudException
+
+logger = logging.getLogger("uvicorn")
+
+check_events_router = APIRouter(tags=["check_events"])
+
+
+# ============================================================================
+# Helper Functions
+# ============================================================================
+
+
+def _is_cloud_user() -> bool:
+    """Check if the current user is connected to Recce Cloud."""
+    ctx = default_context()
+    if ctx is None or ctx.state_loader is None:
+        return False
+    return hasattr(ctx.state_loader, "session_id") and ctx.state_loader.session_id is not None
+
+
+def _get_session_info() -> tuple:
+    """
+    Get organization ID, project ID, and session ID from state loader.
+
+    Returns:
+        tuple: (org_id, project_id, session_id)
+
+    Raises:
+        HTTPException: If not in cloud mode or session info unavailable
+    """
+    if not _is_cloud_user():
+        raise HTTPException(
+            status_code=400,
+            detail="Check events are only available when connected to Recce Cloud.",
+        )
+
+    ctx = default_context()
+    state_loader = ctx.state_loader
+
+    session_id = state_loader.session_id
+
+    # Check if org_id and project_id are cached
+    if hasattr(state_loader, "org_id") and hasattr(state_loader, "project_id"):
+        return state_loader.org_id, state_loader.project_id, session_id
+
+    # Fetch from cloud API
+    api_token = get_recce_api_token() or state_loader.token
+    if not api_token:
+        raise HTTPException(
+            status_code=401,
+            detail="Cannot access Recce Cloud: no API token available.",
+        )
+
+    try:
+        recce_cloud = RecceCloud(api_token)
+        session = recce_cloud.get_session(session_id)
+
+        org_id = session.get("org_id")
+        project_id = session.get("project_id")
+
+        if not org_id or not project_id:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Session {session_id} does not belong to a valid organization or project.",
+            )
+
+        # Cache for future use
+        state_loader.org_id = org_id
+        state_loader.project_id = project_id
+
+        return org_id, project_id, session_id
+
+    except RecceCloudException as e:
+        logger.error(f"Failed to get session info: {e}")
+        raise HTTPException(status_code=e.status_code, detail=str(e.reason))
+
+
+def _get_events_client() -> CheckEventsCloud:
+    """
+    Get the CheckEventsCloud client.
+
+    Returns:
+        CheckEventsCloud: Cloud client for event operations
+
+    Raises:
+        HTTPException: If client cannot be initialized
+    """
+    ctx = default_context()
+    api_token = get_recce_api_token() or ctx.state_loader.token
+
+    if not api_token:
+        raise HTTPException(
+            status_code=401,
+            detail="Cannot access Recce Cloud: no API token available.",
+        )
+
+    return CheckEventsCloud(api_token)
+
+
+# ============================================================================
+# Pydantic Models
+# ============================================================================
+
+
+class CheckEventActorOut(BaseModel):
+    """Actor who performed the event."""
+
+    type: str  # "user", "recce_ai", "preset_system"
+    user_id: Optional[int] = None
+    login: Optional[str] = None
+    fullname: Optional[str] = None
+
+
+class CheckEventOut(BaseModel):
+    """Check event response model."""
+
+    id: str
+    check_id: str
+    event_type: str
+    actor: CheckEventActorOut
+    content: Optional[str] = None
+    old_value: Optional[str] = None
+    new_value: Optional[str] = None
+    is_edited: bool = False
+    is_deleted: bool = False
+    created_at: str
+    updated_at: str
+
+
+class CreateCommentIn(BaseModel):
+    """Request body for creating a comment."""
+
+    content: str
+
+
+class UpdateCommentIn(BaseModel):
+    """Request body for updating a comment."""
+
+    content: str
+
+
+# ============================================================================
+# API Endpoints
+# ============================================================================
+
+
+@check_events_router.get(
+    "/checks/{check_id}/events",
+    status_code=200,
+    response_model=List[CheckEventOut],
+)
+async def list_check_events(check_id: UUID):
+    """
+    List all events for a check in chronological order.
+
+    This endpoint returns all events (comments, state changes, etc.) for the
+    specified check. Events are returned in chronological order (oldest first).
+
+    Args:
+        check_id: The check ID
+
+    Returns:
+        List of CheckEventOut objects
+
+    Raises:
+        400: Not connected to Recce Cloud
+        401: No API token available
+        404: Check not found
+    """
+    try:
+        org_id, project_id, session_id = _get_session_info()
+        client = _get_events_client()
+
+        events = client.list_events(org_id, project_id, session_id, str(check_id))
+        return events
+
+    except RecceCloudException as e:
+        logger.error(f"Failed to list check events: {e}")
+        raise HTTPException(status_code=e.status_code, detail=str(e.reason))
+    except RecceException as e:
+        logger.error(f"Failed to list check events: {e}")
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@check_events_router.get(
+    "/checks/{check_id}/events/{event_id}",
+    status_code=200,
+    response_model=CheckEventOut,
+)
+async def get_check_event(check_id: UUID, event_id: UUID):
+    """
+    Get a specific event by ID.
+
+    Args:
+        check_id: The check ID
+        event_id: The event ID
+
+    Returns:
+        CheckEventOut object
+
+    Raises:
+        400: Not connected to Recce Cloud
+        401: No API token available
+        404: Event not found
+    """
+    try:
+        org_id, project_id, session_id = _get_session_info()
+        client = _get_events_client()
+
+        event = client.get_event(org_id, project_id, session_id, str(check_id), str(event_id))
+        return event
+
+    except RecceCloudException as e:
+        logger.error(f"Failed to get check event: {e}")
+        raise HTTPException(status_code=e.status_code, detail=str(e.reason))
+    except RecceException as e:
+        logger.error(f"Failed to get check event: {e}")
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@check_events_router.post(
+    "/checks/{check_id}/events",
+    status_code=201,
+    response_model=CheckEventOut,
+)
+async def create_comment(check_id: UUID, body: CreateCommentIn):
+    """
+    Create a new comment on a check.
+
+    Args:
+        check_id: The check ID
+        body: Request body containing comment content
+
+    Returns:
+        Created CheckEventOut object
+
+    Raises:
+        400: Not connected to Recce Cloud or invalid content
+        401: No API token available
+        404: Check not found
+    """
+    if not body.content or not body.content.strip():
+        raise HTTPException(status_code=400, detail="Comment content cannot be empty.")
+
+    try:
+        org_id, project_id, session_id = _get_session_info()
+        client = _get_events_client()
+
+        event = client.create_comment(org_id, project_id, session_id, str(check_id), body.content)
+        return event
+
+    except RecceCloudException as e:
+        logger.error(f"Failed to create comment: {e}")
+        raise HTTPException(status_code=e.status_code, detail=str(e.reason))
+    except RecceException as e:
+        logger.error(f"Failed to create comment: {e}")
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@check_events_router.patch(
+    "/checks/{check_id}/events/{event_id}",
+    status_code=200,
+    response_model=CheckEventOut,
+)
+async def update_comment(check_id: UUID, event_id: UUID, body: UpdateCommentIn):
+    """
+    Update an existing comment.
+
+    Only the author or an admin can update a comment.
+
+    Args:
+        check_id: The check ID
+        event_id: The event ID of the comment to update
+        body: Request body containing new comment content
+
+    Returns:
+        Updated CheckEventOut object
+
+    Raises:
+        400: Not connected to Recce Cloud or invalid content
+        401: No API token available
+        403: Not authorized to update this comment
+        404: Comment not found
+    """
+    if not body.content or not body.content.strip():
+        raise HTTPException(status_code=400, detail="Comment content cannot be empty.")
+
+    try:
+        org_id, project_id, session_id = _get_session_info()
+        client = _get_events_client()
+
+        event = client.update_comment(org_id, project_id, session_id, str(check_id), str(event_id), body.content)
+        return event
+
+    except RecceCloudException as e:
+        logger.error(f"Failed to update comment: {e}")
+        raise HTTPException(status_code=e.status_code, detail=str(e.reason))
+    except RecceException as e:
+        logger.error(f"Failed to update comment: {e}")
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@check_events_router.delete(
+    "/checks/{check_id}/events/{event_id}",
+    status_code=204,
+)
+async def delete_comment(check_id: UUID, event_id: UUID):
+    """
+    Delete a comment (soft delete).
+
+    Only the author or an admin can delete a comment. The comment will be
+    marked as deleted but remain in the timeline with a "Comment deleted" indicator.
+
+    Args:
+        check_id: The check ID
+        event_id: The event ID of the comment to delete
+
+    Raises:
+        400: Not connected to Recce Cloud
+        401: No API token available
+        403: Not authorized to delete this comment
+        404: Comment not found
+    """
+    try:
+        org_id, project_id, session_id = _get_session_info()
+        client = _get_events_client()
+
+        client.delete_comment(org_id, project_id, session_id, str(check_id), str(event_id))
+
+    except RecceCloudException as e:
+        logger.error(f"Failed to delete comment: {e}")
+        raise HTTPException(status_code=e.status_code, detail=str(e.reason))
+    except RecceException as e:
+        logger.error(f"Failed to delete comment: {e}")
+        raise HTTPException(status_code=400, detail=str(e))

package/recce-source/recce/apis/check_func.py

@@ -0,0 +1,130 @@
+from datetime import datetime
+from typing import Optional
+
+from fastapi import HTTPException
+
+from recce.apis.run_func import generate_run_name
+from recce.core import default_context
+from recce.models import Check, CheckDAO, RunDAO, RunType
+
+
+def validate_schema_diff_check(params):
+    node_id = params.get("node_id")
+    if node_id is None:
+        raise HTTPException(status_code=400, detail="node_id is required for schema diff")
+    node_name = default_context().get_node_name_by_id(node_id)
+    if node_name is None:
+        raise HTTPException(status_code=400, detail=f"node_id '{node_id}' not found in dbt manifest")
+
+
+def get_node_name_by_id(node_id):
+    node_name = default_context().get_node_name_by_id(node_id)
+    if node_name is None:
+        raise HTTPException(status_code=400, detail=f"node_id '{node_id}' not found in dbt manifest")
+    return node_name
+
+
+def _validate_check(check_type, params):
+    if check_type == RunType.SCHEMA_DIFF:
+        validate_schema_diff_check(params)
+    pass
+
+
+def _get_ref_model(sql_template: str) -> Optional[str]:
+    import re
+
+    pattern = r'\bref\(["\']?(\w+)["\']?\)\s*}}'
+    matches = re.findall(pattern, sql_template)
+    if len(matches) == 1:
+        ref = matches[0]
+        return ref
+
+    return None
+
+
+def _generate_check_name(check_type, params, view_options):
+    now = datetime.utcnow().strftime("%d %b %Y")
+    if check_type == RunType.SCHEMA_DIFF:
+        if params.get("node_id"):
+            nodeIds = params.get("node_id") if isinstance(params.get("node_id"), list) else [params.get("node_id")]
+            if len(nodeIds) == 1:
+                node_name = get_node_name_by_id(nodeIds[0])
+                return f"schema diff of {node_name}".capitalize()
+            else:
+                return f"schema diff of {len(nodeIds)} nodes".capitalize()
+        return f"{'schema diff'.capitalize()} - {now}"
+    elif check_type == RunType.LINEAGE_DIFF:
+        nodes = view_options.get("node_ids") if view_options else params.get("node_ids")
+        if nodes is not None:
+            return f"lineage diff of {len(nodes)} nodes".capitalize()
+        return f"{'lineage diff'.capitalize()} - {now}"
+    else:
+        return f"{'check'.capitalize()} - {now}"
+
+
+def create_check_from_run(
+    run_id, check_name=None, check_description="", check_view_options=None, is_preset=False, is_checked=False
+):
+    if run_id is None:
+        raise ValueError("run_id is required")
+
+    run = RunDAO().find_run_by_id(run_id)
+    if run is None:
+        raise NameError(f"Run '{run_id}' not found")
+
+    run_type = run.type
+    run_params = run.params
+
+    _validate_check(run_type, run_params)
+    name = check_name if check_name is not None else generate_run_name(run)
+    check = Check(
+        name=name,
+        description=check_description,
+        type=run_type,
+        params=run_params,
+        view_options=check_view_options,
+        is_preset=is_preset,
+        is_checked=is_checked,
+    )
+    new_check = CheckDAO().create(check)
+    run.check_id = new_check.check_id
+
+    return new_check
+
+
+def create_check_without_run(
+    check_name, check_description, check_type, params, check_view_options, is_preset=False, is_checked=False
+):
+    name = check_name if check_name is not None else _generate_check_name(check_type, params, check_view_options)
+    check = Check(
+        name=name,
+        description=check_description,
+        type=check_type,
+        params=params,
+        view_options=check_view_options,
+        is_preset=is_preset,
+        is_checked=is_checked,
+    )
+    new_check = CheckDAO().create(check)
+    return new_check
+
+
+def purge_preset_checks():
+    checks = CheckDAO().list()
+    for check in checks:
+        if check.is_preset:
+            related_runs = RunDAO().list_by_check_id(check.check_id)
+            for run in related_runs:
+                RunDAO().delete(run.run_id)
+            CheckDAO().delete(check.check_id)
+
+
+def export_persistent_state():
+    ctx = default_context()
+    state_loader = ctx.state_loader
+    if state_loader is not None:
+        is_conflict = state_loader.check_conflict()
+        if is_conflict:
+            ctx.sync_state("merge")
+        else:
+            ctx.sync_state("overwrite")

package/recce-source/recce/apis/run_api.py

@@ -0,0 +1,130 @@
+import asyncio
+from typing import List, Optional
+from uuid import UUID
+
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel
+
+from recce.apis.run_func import cancel_run, materialize_run_results, submit_run
+from recce.event import log_api_event
+from recce.exceptions import RecceException
+from recce.models import RunDAO
+
+run_router = APIRouter(tags=["run"])
+
+
+class CreateRunIn(BaseModel):
+    type: str
+    params: dict
+    check_id: Optional[str] = None
+    nowait: Optional[bool] = False
+    track_props: Optional[dict] = None
+
+
+@run_router.post("/runs", status_code=201)
+async def create_run_handler(input: CreateRunIn):
+    log_api_event(
+        "create_run",
+        dict(
+            type=input.type,
+            track_props=input.track_props,
+        ),
+    )
+    try:
+        run, future = submit_run(input.type, input.params)
+    except RecceException as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+    if input.nowait:
+        return run
+    else:
+        run.result = await future
+        return run
+
+
+@run_router.post("/runs/{run_id}/cancel")
+async def cancel_run_handler(run_id: UUID):
+    try:
+        cancel_run(run_id)
+    except NotImplementedError:
+        pass
+
+
+@run_router.get("/runs/{run_id}/wait")
+async def wait_run_handler(run_id: UUID, timeout: int = Query(None, description="Maximum number of seconds to wait")):
+    run = RunDAO().find_run_by_id(run_id)
+    if run is None:
+        raise HTTPException(status_code=404, detail="Not Found")
+
+    start_time = asyncio.get_event_loop().time()
+    while run.result is None and run.error is None:
+        await asyncio.sleep(1)
+        if timeout is not None and (asyncio.get_event_loop().time() - start_time) > timeout:
+            break
+    return run
+
+
+@run_router.get("/runs", status_code=200)
+async def list_run_handler():
+    runs = RunDAO().list() or []
+
+    result = [
+        {
+            "run_id": run.run_id,
+            "run_at": run.run_at,
+            "name": run.name,
+            "type": run.type,
+            "params": run.params,
+            "status": run.status,
+            "check_id": run.check_id,
+        }
+        for run in runs
+    ]
+
+    # sort by run_at
+    result = sorted(result, key=lambda x: x["run_at"], reverse=True)
+
+    return result
+
+
+class SearchRunsIn(BaseModel):
+    type: str
+    params: dict
+    limit: Optional[int] = None
+
+
+@run_router.post("/runs/search", status_code=200)
+async def search_runs_handler(search: SearchRunsIn):
+    runs = RunDAO().list()
+
+    result = []
+    for run in runs:
+        if run.type.value != search.type:
+            continue
+        if not all(search.params[key] == run.params.get(key) for key in search.params.keys()):
+            continue
+
+        result.append(run)
+
+    if search.limit:
+        return result[-search.limit :]
+
+    return result
+
+
+class AggregateRunsIn(BaseModel):
+    class AggregateFilter(BaseModel):
+        nodes: Optional[List[str]] = None
+
+    filter: Optional[AggregateFilter] = None
+
+
+@run_router.post("/runs/aggregate", status_code=200)
+async def aggregate_runs_handler(input: AggregateRunsIn):
+    try:
+        runs = RunDAO().list()
+        nodes = input.filter.nodes if input.filter and input.filter.nodes else None
+        result = materialize_run_results(runs, nodes=nodes)
+        return result
+    except Exception as e:
+        raise HTTPException(status_code=405, detail=str(e))