@datarecce/ui 0.1.40 → 0.2.0

package/recce-source/recce/models/types.py

@@ -1,218 +0,0 @@
-import uuid
-from datetime import datetime, timezone
-from enum import Enum
-from typing import Dict, List, Literal, Optional, Set
-
-from pydantic import UUID4, BaseModel, Field
-
-from recce.util.pydantic_model import pydantic_model_dump
-
-
-class RunType(Enum):
-    SIMPLE = "simple"
-    QUERY = "query"
-    QUERY_BASE = "query_base"
-    QUERY_DIFF = "query_diff"
-    VALUE_DIFF = "value_diff"
-    VALUE_DIFF_DETAIL = "value_diff_detail"
-    SCHEMA_DIFF = "schema_diff"
-    PROFILE = "profile"
-    PROFILE_DIFF = "profile_diff"
-    ROW_COUNT = "row_count"
-    ROW_COUNT_DIFF = "row_count_diff"
-    LINEAGE_DIFF = "lineage_diff"
-    TOP_K_DIFF = "top_k_diff"
-    HISTOGRAM_DIFF = "histogram_diff"
-
-    def __str__(self):
-        return self.value
-
-
-class RunProgress(BaseModel):
-    message: str
-    percentage: float
-
-
-class RunStatus(Enum):
-    FINISHED = "finished"
-    FAILED = "failed"
-    CANCELLED = "cancelled"
-    RUNNING = "running"
-
-
-class Run(BaseModel):
-    type: RunType
-    name: Optional[str] = None
-    params: Optional[dict] = None
-    check_id: Optional[UUID4] = None
-    result: Optional[dict] = None
-    error: Optional[str] = None
-    status: Optional[RunStatus] = None
-    progress: Optional[RunProgress] = None
-    run_id: UUID4 = Field(default_factory=uuid.uuid4)
-    run_at: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
-
-    def __init__(self, **data):
-        type = data.get("type")
-
-        if "result" in data and data["result"] is not None:
-            result = data.get("result")
-
-            if type in [RunType.QUERY.value, RunType.QUERY_BASE.value]:
-                from recce.tasks.query import QueryResult
-
-                data["result"] = pydantic_model_dump(QueryResult(**result))
-            elif type == RunType.QUERY_DIFF.value:
-                from recce.tasks.query import QueryDiffResult
-
-                data["result"] = pydantic_model_dump(QueryDiffResult(**result))
-            elif type == RunType.PROFILE.value:
-                from recce.tasks.profile import ProfileResult
-
-                data["result"] = pydantic_model_dump(ProfileResult(**result))
-            elif type == RunType.PROFILE_DIFF.value:
-                from recce.tasks.profile import ProfileDiffResult
-
-                data["result"] = pydantic_model_dump(ProfileDiffResult(**result))
-            elif type == RunType.VALUE_DIFF.value:
-                from recce.tasks.valuediff import ValueDiffResult
-
-                data["result"] = pydantic_model_dump(ValueDiffResult(**result))
-            elif type == RunType.VALUE_DIFF_DETAIL.value:
-                from recce.tasks.valuediff import ValueDiffDetailResult
-
-                data["result"] = pydantic_model_dump(ValueDiffDetailResult(**result))
-
-        super().__init__(**data)
-
-
-class Check(BaseModel):
-    name: str
-    description: Optional[str] = None
-    type: RunType
-    params: Optional[dict] = {}
-    view_options: Optional[dict] = {}
-    check_id: UUID4 = Field(default_factory=uuid.uuid4)
-    session_id: Optional[UUID4] = Field(default=None)
-    is_checked: bool = False
-    is_preset: bool = False
-    created_by: Optional[str] = None
-    updated_by: Optional[str] = None
-    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc).replace(microsecond=0))
-    updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc).replace(microsecond=0))
-
-    def merge(self, other) -> bool:
-        if not self.is_preset and not other.is_preset and self.check_id != other.check_id:
-            raise ValueError(f"check_id mismatch: {self.check_id} != {other.check_id}")
-
-        if self.updated_at and other.updated_at and self.updated_at >= other.updated_at:
-            return False
-
-        self.name = other.name
-        self.description = other.description
-        self.type = other.type
-        self.params = other.params
-        self.view_options = other.view_options
-        self.is_checked = other.is_checked
-        self.is_preset = other.is_preset
-        self.updated_at = other.updated_at
-        return True
-
-
-ChangeStatus = Literal[
-    "added",
-    "removed",
-    "modified",
-]
-ChangeCategory = Literal[
-    "breaking",
-    "non_breaking",
-    "partial_breaking",
-    "unknown",
-]
-
-
-class NodeChange(BaseModel):
-    category: ChangeCategory
-    columns: Optional[dict[str, ChangeStatus]] = None
-
-
-class NodeDiff(BaseModel):
-    change_status: ChangeStatus
-    change: Optional[NodeChange] = None  # Only available if change_status is 'modified'
-
-
-class LineageDiff(BaseModel):
-    base: dict
-    current: dict
-    diff: dict[str, NodeDiff]
-
-
-# Column Level Linage
-class CllColumnDep(BaseModel):
-    node: str
-    column: str
-
-
-class CllColumn(BaseModel):
-    id: Optional[str] = None
-    table_id: Optional[str] = None
-    name: Optional[str] = None
-
-    # data type
-    type: Optional[str] = None
-
-    # transformation type
-    transformation_type: Literal["source", "passthrough", "renamed", "derived", "unknown"] = "unknown"
-
-    # change analysis
-    change_status: Optional[ChangeStatus] = None
-
-    # column-to-column dependencies
-    depends_on: List[CllColumnDep] = Field(default_factory=list)
-
-
-class CllNode(BaseModel):
-    id: str
-    name: str
-    package_name: str
-    resource_type: str
-    raw_code: Optional[str] = None
-    source_name: Optional[str] = None
-
-    # change analysis
-    change_status: Optional[ChangeStatus] = None
-    change_category: Optional[ChangeCategory] = None
-
-    # Column to column dependencies
-    columns: Dict[str, CllColumn] = Field(default_factory=dict)
-
-    # If the node is impacted. Only used if option 'change_analysis' is set
-    impacted: Optional[bool] = None
-
-    @classmethod
-    def build_cll_node(cls, manifest, resource_key, node_id) -> Optional["CllNode"]:
-        resources = getattr(manifest, resource_key)
-        if node_id not in resources:
-            return None
-        n = resources[node_id]
-        if resource_key == "nodes" and n.resource_type not in ["model", "seed", "snapshot"]:
-            return None
-        cll_node = CllNode(
-            id=n.unique_id,
-            name=n.name,
-            package_name=n.package_name,
-            resource_type=n.resource_type,
-        )
-        if resource_key == "sources":
-            cll_node.source_name = n.source_name
-        elif resource_key == "nodes":
-            cll_node.raw_code = n.raw_code
-        return cll_node
-
-
-class CllData(BaseModel):
-    nodes: Dict[str, CllNode] = Field(default_factory=dict)
-    columns: Dict[str, CllColumn] = Field(default_factory=dict)
-    parent_map: Dict[str, Set[str]] = Field(default_factory=dict)
-    child_map: Dict[str, Set[str]] = Field(default_factory=dict)

package/recce-source/recce/pull_request.py

@@ -1,124 +0,0 @@
-import json
-import os
-from typing import Optional, Union
-
-import requests
-from pydantic import BaseModel
-
-from recce.git import hosting_repo
-from recce.github import recce_pr_information
-from recce.util.pydantic_model import pydantic_model_dump
-
-
-class PullRequestInfo(BaseModel):
-    id: Optional[Union[int, str]] = None
-    title: Optional[str] = None
-    url: Optional[str] = None
-    branch: Optional[str] = None
-    base_branch: Optional[str] = None
-    repository: Optional[str] = None
-
-    def to_dict(self):
-        return pydantic_model_dump(self)
-
-
-def fetch_pr_metadata(**kwargs):
-    pr_info = PullRequestInfo()
-
-    # fetch from github action event path
-    metadata = fetch_pr_metadata_from_event_path()
-    if metadata is not None:
-        pr_info.id = metadata.get("github_pr_id")
-        pr_info.url = metadata.get("github_pr_url")
-        pr_info.title = metadata.get("github_pr_title")
-        pr_info.repository = metadata.get("github_repository")
-    else:
-        repo = hosting_repo()
-        pr, message = recce_pr_information(github_token=kwargs.get("github_token"))
-        if kwargs.get("cloud") and message:
-            print(message)
-
-        if pr:
-            pr_info.repository = repo
-            pr_info.id = pr.number
-            pr_info.title = pr.title
-            pr_info.url = pr.html_url
-            pr_info.base_branch = pr.base.ref
-            pr_info.branch = pr.head.ref
-
-    # fetch from cli arguments
-    if pr_info.url is None and "github_pull_request_url" in kwargs:
-        pr_info.url = kwargs.get("github_pull_request_url")
-
-    if pr_info.branch is None:
-        pr_info.branch = kwargs.get("git_current_branch")
-
-    if pr_info.base_branch is None:
-        pr_info.base_branch = kwargs.get("git_base_branch")
-
-    # fetch from env
-    if pr_info.url is None:
-        pr_info.url = os.getenv("RECCE_PR_URL")
-
-    return pr_info
-
-
-def fetch_pr_metadata_from_event_path() -> Optional[dict]:
-    """
-    If recce run is running in a GitHub Action, this function will return the pull request metadata.
-
-    Example:
-    {
-        "github_pr_id": 1,
-        "github_pr_url": "https://github.com/xyz/abc/pull/1,
-        "github_pr_title": "Update README.md",
-        "github_repository": "xyz/abc"
-    }
-
-    :return: dict
-    """
-
-    # get the event json from the path in GITHUB_EVENT_PATH
-    event_path = os.getenv("GITHUB_EVENT_PATH")
-    github_repository = os.getenv("GITHUB_REPOSITORY")
-    if event_path:
-        try:
-            with open(event_path, "r", encoding="utf-8") as event_file:
-                event_data = json.load(event_file)
-
-            pr_id = event_data["number"]
-            if event_data.get("pull_request"):
-                pull_request_data = event_data["pull_request"]
-                pr_url = pull_request_data["_links"]["html"]["href"]
-                pr_api = pull_request_data["_links"]["self"]["href"]
-                pr_title = _fetch_pr_title(pr_api)
-                return dict(
-                    github_pr_id=pr_id,
-                    github_pr_url=pr_url,
-                    github_pr_title=pr_title,
-                    github_repository=github_repository,
-                )
-            else:
-                print("Not a pull request event, skip.")
-        except Exception as e:
-            print("Cannot parse github action event", e)
-    return None
-
-
-def _fetch_pr_title(endpoint) -> Optional[str]:
-    github_token = os.getenv("GITHUB_TOKEN")
-
-    if github_token is None:
-        return None
-
-    try:
-        headers = {"Authorization": f"Bearer {github_token}"}
-        response = requests.get(endpoint, headers=headers)
-
-        if response.status_code == 200:
-            pull_request_data = response.json()
-            return pull_request_data.get("title")
-    except Exception as e:
-        print("Cannot fetch PR title: ", e)
-
-    return None