@datarecce/ui 0.1.41 → 0.2.0

package/recce-source/recce/tasks/core.py

@@ -1,134 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import List, Literal, Optional, Union
-
-from pydantic import BaseModel
-
-from recce.core import default_context
-from recce.exceptions import RecceCancelException
-from recce.models import Check, Run
-from recce.util.pydantic_model import pydantic_model_dump
-
-
-class Task(ABC):
-    def __init__(self):
-        self.is_cancelled = False
-        self._progress_listener = None
-
-    @property
-    def progress_listener(self):
-        return self._progress_listener
-
-    @progress_listener.setter
-    def progress_listener(self, value):
-        self._progress_listener = value
-
-    @abstractmethod
-    def execute(self):
-        """
-        Execute the run.
-        """
-        raise NotImplementedError()
-
-    def cancel(self):
-        """
-        Cancel the run. Subclass should override this method to implement the cancellation logic.
-        """
-        self.is_cancelled = True
-
-    def check_cancel(self):
-        """
-        Check if the run is canceled. If so, raise an exception.
-        """
-
-        if self.is_cancelled:
-            raise RecceCancelException()
-
-    def update_progress(self, message=None, percentage=None):
-        if self.progress_listener is not None:
-            self.progress_listener(message=message, percentage=percentage)
-
-
-class TaskResultDiffer(ABC):
-    related_node_ids: List[str] = None
-
-    def __init__(self, run: Run):
-        # Dump the result to a dict when we create summary from recce run
-        if isinstance(run.result, BaseModel):
-            run.result = pydantic_model_dump(run.result)
-        self.run = run
-        self.changes = self._check_result_changed_fn(run.result) if run.result else None
-        self.related_node_ids = self._get_related_node_ids()
-        self.changed_nodes = self._get_changed_nodes()
-
-    @staticmethod
-    def diff(base, current):
-        from deepdiff import DeepDiff
-
-        diff = DeepDiff(base, current, ignore_order=True)
-        return diff if diff else None
-
-    @staticmethod
-    def get_node_id_by_name(name):
-        node = default_context().adapter.get_node_by_name(name)
-        return node.unique_id if node else None
-
-    @staticmethod
-    def get_node_ids_by_selector(
-        select: Optional[str] = None,
-        exclude: Optional[str] = None,
-        packages: Optional[list[str]] = None,
-        view_mode: Optional[Literal["all", "changed_models"]] = None,
-    ) -> List[str]:
-        nodes = default_context().adapter.select_nodes(
-            select=select, exclude=exclude, packages=packages, view_mode=view_mode
-        )
-        return [node for node in nodes if not node.startswith("test.")]
-
-    @abstractmethod
-    def _check_result_changed_fn(self, result):
-        """
-        Check if the result is changed.
-        Should be implemented by subclass.
-        """
-        raise NotImplementedError()
-
-    def _get_related_node_ids(self) -> Union[List[str], None]:
-        """
-        Get the related node ids.
-        Should be implemented by subclass.
-        """
-        params = self.run.params
-        if params.get("model"):
-            return [TaskResultDiffer.get_node_id_by_name(params.get("model"))]
-        elif params.get("node_names"):
-            names = params.get("node_names", [])
-            return [TaskResultDiffer.get_node_id_by_name(name) for name in names]
-        else:
-            # No related node ids in the params
-            return None
-
-    def _get_changed_nodes(self) -> Union[List[str], None]:
-        """
-        Get the changed node ids.
-        Should be implemented by subclass.
-        """
-        return None
-
-
-class CheckValidator:
-    def __init__(self):
-        pass
-
-    def validate(self, check: dict):
-        try:
-            check = Check(**check)
-        except Exception as e:
-            raise ValueError(f"Invalid check format. {str(e)}")
-
-        self.validate_check(check)
-
-    def validate_check(self, check: Check):
-        """
-        Validate the check. This is supposed to be overridden by subclass. Throw ValueError if the check is invalid.
-        """
-        pass

package/recce-source/recce/tasks/dataframe.py

@@ -1,170 +0,0 @@
-import json
-import typing as t
-from decimal import Decimal
-from enum import Enum
-
-if t.TYPE_CHECKING:
-    import agate
-    import pandas
-from pydantic import BaseModel, Field
-
-
-class DataFrameColumnType(Enum):
-    NUMBER = "number"
-    INTEGER = "integer"
-    TEXT = "text"
-    BOOLEAN = "boolean"
-    DATE = "date"
-    DATETIME = "datetime"
-    TIMEDELTA = "timedelta"
-    UNKNOWN = "unknown"
-
-    @classmethod
-    def from_string(cls, type_str: str) -> "DataFrameColumnType":
-        """Convert string to DataFrameColumnType enum.
-
-        Args:
-            type_str: String representation of the type (e.g., "integer", "text")
-
-        Returns:
-            DataFrameColumnType enum value
-        """
-        type_str = type_str.lower().strip()
-        try:
-            return cls(type_str)
-        except ValueError:
-            return cls.UNKNOWN
-
-
-class DataFrameColumn(BaseModel):
-    key: t.Optional[str] = None
-    name: str
-    type: DataFrameColumnType
-
-    def __init__(self, **data):
-        """Initialize DataFrameColumn, auto-setting key=name if key is missing."""
-        if "key" not in data or data["key"] is None:
-            data["key"] = data.get("name")
-        super().__init__(**data)
-
-
-class DataFrame(BaseModel):
-    columns: t.List[DataFrameColumn]
-    data: t.List[tuple]
-    limit: t.Optional[int] = Field(None, description="Limit the number of rows returned")
-    more: t.Optional[bool] = Field(None, description="Whether there are more rows to fetch")
-
-    @staticmethod
-    def from_agate(table: "agate.Table", limit: t.Optional[int] = None, more: t.Optional[bool] = None):
-        from recce.adapter.dbt_adapter import dbt_version
-
-        if dbt_version < "v1.8":
-            import dbt.clients.agate_helper as agate_helper
-        else:
-            import dbt_common.clients.agate_helper as agate_helper
-
-        import agate
-
-        columns = []
-
-        for col_name, col_type in zip(table.column_names, table.column_types):
-
-            has_integer = hasattr(agate_helper, "Integer")
-
-            if isinstance(col_type, agate.Number):
-                col_type = DataFrameColumnType.NUMBER
-            elif isinstance(col_type, agate.Text):
-                col_type = DataFrameColumnType.TEXT
-            elif isinstance(col_type, agate.Boolean):
-                col_type = DataFrameColumnType.BOOLEAN
-            elif isinstance(col_type, agate.Date):
-                col_type = DataFrameColumnType.DATE
-            elif isinstance(col_type, agate.DateTime):
-                col_type = DataFrameColumnType.DATETIME
-            elif isinstance(col_type, agate.TimeDelta):
-                col_type = DataFrameColumnType.TIMEDELTA
-            elif has_integer and isinstance(col_type, agate_helper.Integer):
-                col_type = DataFrameColumnType.INTEGER
-            else:
-                col_type = DataFrameColumnType.UNKNOWN
-            columns.append(DataFrameColumn(key=col_name, name=col_name, type=col_type))
-
-        def _row_values(row):
-            # If the value is Decimal, check if it's finite. If not, convert it to float(xxx) (GitHub issue #476)
-            return tuple([float(v) if isinstance(v, Decimal) and not v.is_finite() else v for v in row.values()])
-
-        data = [_row_values(row) for row in table.rows]
-        df = DataFrame(
-            columns=columns,
-            data=data,
-            limit=limit,
-            more=more,
-        )
-        return df
-
-    @staticmethod
-    def from_pandas(pandas_df: "pandas.DataFrame", limit: t.Optional[int] = None, more: t.Optional[bool] = None):
-        columns = []
-        for column in pandas_df.columns:
-            dtype = pandas_df[column].dtype
-            if dtype == "int64":
-                col_type = DataFrameColumnType.INTEGER
-            elif dtype == "float64":
-                col_type = DataFrameColumnType.NUMBER
-            elif dtype == "object":
-                col_type = DataFrameColumnType.TEXT
-            elif dtype == "bool":
-                col_type = DataFrameColumnType.BOOLEAN
-            else:
-                col_type = DataFrameColumnType.UNKNOWN
-            columns.append(DataFrameColumn(name=column, type=col_type))
-
-        s = pandas_df.to_json(orient="values")
-        data = json.loads(s)
-
-        df = DataFrame(
-            columns=columns,
-            data=data,
-            limit=limit,
-            more=more,
-        )
-        return df
-
-    @staticmethod
-    def from_data(
-        columns: t.Dict[str, str],
-        data: t.List[tuple],
-        limit: t.Optional[int] = None,
-        more: t.Optional[bool] = None,
-    ):
-        """Create a DataFrame from columns and data directly.
-
-        Args:
-            columns: Dict defining the schema where keys are column names and values are type strings.
-                     Type strings can be: "number", "integer", "text", "boolean", "date", "datetime", "timedelta"
-            data: List of rows (each row is a list/tuple/sequence of values)
-            limit: Optional limit on the number of rows returned
-            more: Optional flag indicating whether there are more rows to fetch
-
-        Returns:
-            DataFrame instance
-
-        Examples:
-            # Using simple dict format
-            columns = {"idx": "integer", "name": "text", "impacted": "boolean"}
-            data = [[0, "model_a", True], [1, "model_b", False]]
-            df = DataFrame.from_data(columns, data)
-        """
-        # Convert dict columns to DataFrameColumn objects
-        processed_columns = []
-        for key, type_str in columns.items():
-            col_type = DataFrameColumnType.from_string(type_str)
-            processed_columns.append(DataFrameColumn(key=key, name=key, type=col_type))
-
-        df = DataFrame(
-            columns=processed_columns,
-            data=data,
-            limit=limit,
-            more=more,
-        )
-        return df