@datarecce/ui 0.1.40 → 0.2.0

package/recce-source/recce/adapter/dbt_adapter/dbt_version.py

@@ -1,42 +0,0 @@
-class DbtVersion:
-
-    def __init__(self):
-        from dbt import version as dbt_version
-
-        dbt_version = self.parse(dbt_version.__version__)
-        if dbt_version.is_prerelease:
-            dbt_version = self.parse(dbt_version.base_version)
-        self.dbt_version = dbt_version
-
-    @staticmethod
-    def parse(version: str):
-        from packaging import version as v
-
-        return v.parse(version)
-
-    def as_version(self, other):
-        from packaging.version import Version
-
-        if isinstance(other, Version):
-            return other
-        if isinstance(other, str):
-            return self.parse(other)
-        return self.parse(str(other))
-
-    def __ge__(self, other):
-        return self.dbt_version >= self.as_version(other)
-
-    def __gt__(self, other):
-        return self.dbt_version > self.as_version(other)
-
-    def __lt__(self, other):
-        return self.dbt_version < self.as_version(other)
-
-    def __le__(self, other):
-        return self.dbt_version <= self.as_version(other)
-
-    def __eq__(self, other):
-        return self.dbt_version.release[:2] == self.as_version(other).release[:2]
-
-    def __str__(self):
-        return ".".join([str(x) for x in list(self.dbt_version.release)])

package/recce-source/recce/adapter/sqlmesh_adapter.py

@@ -1,141 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-from typing import Dict, Optional, Type
-
-import pandas as pd
-from sqlglot import Expression, parse_one, select
-from sqlglot.optimizer import traverse_scope
-from sqlmesh.core.context import Context as SqlmeshContext
-from sqlmesh.core.environment import Environment
-from sqlmesh.core.state_sync import StateReader
-
-from recce.adapter.base import BaseAdapter
-from recce.models import RunType
-from recce.tasks import QueryDiffTask, QueryTask, RowCountDiffTask, Task
-
-sqlmesh_supported_registry: Dict[RunType, Type[Task]] = {
-    RunType.QUERY: QueryTask,
-    RunType.QUERY_DIFF: QueryDiffTask,
-    RunType.ROW_COUNT_DIFF: RowCountDiffTask,
-}
-
-
-@dataclass
-class SqlmeshAdapter(BaseAdapter):
-    context: SqlmeshContext
-    base_env: Environment
-    curr_env: Environment
-
-    def support_tasks(self):
-        return {run_type.value: True for run_type in sqlmesh_supported_registry}
-
-    def get_lineage(self, base: Optional[bool] = False):
-        state_reader: StateReader = self.context.state_reader
-        env = self.base_env if base else self.curr_env
-        nodes = {}
-        parent_map = {}
-
-        for snapshot in state_reader.get_snapshots(env.snapshots, hydrate_seeds=True).values():
-
-            if snapshot.node_type.lower() != "model":
-                continue
-
-            model = snapshot.model
-            if not model:
-                continue
-
-            node = {
-                "unique_id": model.name,
-                "name": model.name,
-                "resource_type": snapshot.node_type.lower(),
-                "checksum": {"checksum": snapshot.fingerprint.data_hash},
-            }
-
-            columns = {}
-            for column, type in model.columns_to_types.items():
-                columns[column] = {"name": column, "type": str(type)}
-            node["columns"] = columns
-
-            nodes[snapshot.name] = node
-            parents = [snapshotId.name for snapshotId in snapshot.parents]
-            parent_map[snapshot.name] = parents
-
-        manifest_metadata = {}
-        catalog_metadata = {}
-
-        return dict(
-            parent_map=parent_map,
-            nodes=nodes,
-            manifest_metadata=manifest_metadata,
-            catalog_metadata=catalog_metadata,
-        )
-
-    def get_model(self, model_id: str, base=False):
-        raise NotImplementedError()
-
-    def get_node_name_by_id(self, unique_id):
-        model = self.context.models.get(unique_id)
-        if model is None:
-            return None
-        return model.name
-
-    @classmethod
-    def load(cls, **kwargs):
-        sqlmesh_envs = kwargs.get("sqlmesh_envs")
-        if sqlmesh_envs is None:
-            raise Exception("'--sqlmesh-envs SOURCE:TARGET' is required")
-
-        envs = sqlmesh_envs.split(":")
-        if len(envs) != 2:
-            raise Exception('sqlmesh_envs must be in the format of "SOURCE:TARGET"')
-
-        sqlmesh_config = kwargs.get("sqlmesh_config", None)
-        context = SqlmeshContext(config=sqlmesh_config)
-        base_env = context.state_reader.get_environment(envs[0])
-        curr_env = context.state_reader.get_environment(envs[1])
-        if base_env is None:
-            raise Exception(f"Source environment {envs[0]} not found")
-        if curr_env is None:
-            raise Exception(f"Target environment {envs[1]} not found")
-
-        return cls(context=context, base_env=base_env, curr_env=curr_env)
-
-    def replace_virtual_tables(self, sql: t.Union[Expression, str], base: bool = None) -> Expression:
-        """
-        Replace virtual tables based on the env name.
-
-        Args:
-            sql: SQL expression to replace virtual tables
-            base: True: replace virtual tables with base env, False: replace virtual tables with current env, None: no replacement
-        """
-        if isinstance(sql, str):
-            expression = parse_one(sql, dialect=self.context.default_dialect)
-        else:
-            expression = sql
-
-        if base is not None:
-            env = self.base_env if base else self.curr_env
-            if env.name != "prod":
-                model_names = [model.name for model in self.context.models.values()]
-                for scope in traverse_scope(expression):
-                    for table in scope.tables:
-                        if f"{table.db}.{table.name}" in model_names:
-                            table.args["db"] = f"{table.args['db']}__{env.name}"
-
-        return expression
-
-    def fetchdf_with_limit(
-        self, sql: t.Union[Expression, str], base: Optional[bool] = None, limit: Optional[int] = None
-    ) -> (pd.DataFrame, bool):
-        expression = self.replace_virtual_tables(sql, base=base)
-        if limit:
-            expression = (
-                select("*").from_("__QUERY").with_("__QUERY", as_=expression).limit(limit + 1 if limit else None)
-            )
-        df = self.context.fetchdf(expression)
-        if limit and len(df) > limit:
-            df = df.head(limit)
-            more = True
-        else:
-            more = False
-        return df, more

package/recce-source/recce/apis/check_api.py

@@ -1,203 +0,0 @@
-from typing import Optional
-from uuid import UUID
-
-from fastapi import APIRouter, BackgroundTasks, HTTPException
-from pydantic import BaseModel
-
-from recce.apis.check_func import (
-    create_check_from_run,
-    create_check_without_run,
-    export_persistent_state,
-)
-from recce.apis.run_func import submit_run
-from recce.event import log_api_event
-from recce.exceptions import RecceException
-from recce.models import Check, CheckDAO, Run, RunDAO, RunType
-
-check_router = APIRouter(tags=["check"])
-
-
-class CreateCheckIn(BaseModel):
-    name: Optional[str] = None
-    description: str = ""
-    run_id: Optional[str] = None
-    type: Optional[RunType] = (None,)
-    params: Optional[dict] = None
-    view_options: Optional[dict] = None
-    track_props: Optional[dict] = None
-
-
-class CheckOut(BaseModel):
-    check_id: UUID
-    name: str
-    description: str
-    type: str
-    params: Optional[dict] = None
-    view_options: Optional[dict] = None
-    is_checked: bool = False
-    is_preset: bool = False
-    last_run: Optional[Run] = None
-
-    @classmethod
-    def from_check(cls, check: Check):
-        check_related_runs = RunDAO().list_by_check_id(check.check_id)
-        last_run = check_related_runs[-1] if len(check_related_runs) > 0 else None
-        return CheckOut(
-            check_id=check.check_id,
-            name=check.name,
-            description=check.description,
-            type=check.type.value,
-            params=check.params,
-            view_options=check.view_options,
-            is_checked=check.is_checked,
-            is_preset=check.is_preset,
-            last_run=last_run,
-        )
-
-
-@check_router.post("/checks", status_code=201, response_model=CheckOut)
-async def create_check(check_in: CreateCheckIn, background_tasks: BackgroundTasks):
-    try:
-        if check_in.run_id is not None:
-            check = create_check_from_run(
-                check_in.run_id,
-                check_name=check_in.name,
-                check_description=check_in.description,
-                check_view_options=check_in.view_options,
-            )
-        else:
-            check = create_check_without_run(
-                check_name=check_in.name,
-                check_description=check_in.description,
-                check_type=check_in.type,
-                params=check_in.params,
-                check_view_options=check_in.view_options,
-            )
-        log_api_event(
-            "create_check",
-            dict(
-                type=str(check.type),
-                track_props=check_in.track_props,
-            ),
-        )
-    except NameError as e:
-        raise HTTPException(status_code=404, detail=str(e))
-    except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
-
-    background_tasks.add_task(export_persistent_state)
-    return CheckOut.from_check(check)
-
-
-class RunCheckIn(BaseModel):
-    nowait: Optional[bool] = False
-
-
-@check_router.post("/checks/{check_id}/run", status_code=201, response_model=Run)
-async def run_check_handler(check_id: UUID, input: RunCheckIn):
-    check = CheckDAO().find_check_by_id(check_id)
-    if not check:
-        raise HTTPException(status_code=404, detail=f"Check ID '{check_id}' not found")
-
-    try:
-        log_api_event(
-            "rerun_check",
-            dict(
-                type=str(check.type),
-                rerun=True,
-            ),
-        )
-        run, future = submit_run(check.type, check.params, check_id=check_id)
-    except RecceException as e:
-        raise HTTPException(status_code=400, detail=str(e))
-
-    if input.nowait:
-        return run
-    else:
-        run.result = await future
-        return run
-
-
-@check_router.get("/checks", status_code=200, response_model=list[CheckOut], response_model_exclude_none=True)
-async def list_checks_handler():
-    checks = []
-
-    for check in CheckDAO().list():
-        checks.append(CheckOut.from_check(check))
-
-    return checks
-
-
-@check_router.get("/checks/{check_id}", status_code=200, response_model=CheckOut)
-async def get_check_handler(check_id: UUID):
-    check = CheckDAO().find_check_by_id(check_id)
-    if check is None:
-        raise HTTPException(status_code=404, detail="Not Found")
-
-    runs = RunDAO().list_by_check_id(check_id)
-    last_run = runs[-1] if len(runs) > 0 else None
-
-    out = CheckOut.from_check(check)
-    out.last_run = last_run
-    return out
-
-
-class PatchCheckIn(BaseModel):
-    name: Optional[str] = None
-    description: Optional[str] = None
-    params: Optional[dict] = None
-    view_options: Optional[dict] = None
-    is_checked: Optional[bool] = None
-
-
-@check_router.patch("/checks/{check_id}", status_code=200, response_model=CheckOut, response_model_exclude_none=True)
-async def update_check_handler(check_id: UUID, patch: PatchCheckIn, background_tasks: BackgroundTasks):
-    check = CheckDAO().update_check_by_id(check_id, patch)
-    if check is None:
-        raise HTTPException(status_code=404, detail="Not Found")
-
-    background_tasks.add_task(export_persistent_state)
-    return CheckOut.from_check(check)
-
-
-class DeleteCheckOut(BaseModel):
-    check_id: UUID
-
-
-@check_router.delete("/checks/{check_id}", status_code=200, response_model=DeleteCheckOut)
-async def delete_handler(check_id: UUID, background_tasks: BackgroundTasks):
-    CheckDAO().delete(check_id)
-    background_tasks.add_task(export_persistent_state)
-    return DeleteCheckOut(check_id=check_id)
-
-
-class ReorderChecksIn(BaseModel):
-    source: int
-    destination: int
-
-
-@check_router.post("/checks/reorder", status_code=200)
-async def reorder_handler(order: ReorderChecksIn):
-    try:
-        CheckDAO().reorder(order.source, order.destination)
-    except RecceException as e:
-        raise HTTPException(status_code=400, detail=e.message)
-
-
-@check_router.post("/checks/{check_id}/mark-as-preset", status_code=204)
-async def mark_as_preset_check_handler(check_id: UUID, background_tasks: BackgroundTasks):
-    """
-    Mark an existing check as a preset check (cloud users only).
-
-    This creates a preset check from the specified check.
-    Only available for users with cloud mode enabled.
-
-    Returns:
-        204 No Content: Successfully marked check as preset
-        400 Bad Request: Error with detail message (e.g., not in cloud mode, check not found)
-    """
-    try:
-        CheckDAO().mark_as_preset_check(check_id)
-        background_tasks.add_task(export_persistent_state)
-    except RecceException as e:
-        raise HTTPException(status_code=400, detail=e.message)