@datarecce/ui 0.1.29 → 0.1.31

package/recce-source/recce/models/types.py

@@ -0,0 +1,218 @@
+import uuid
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Dict, List, Literal, Optional, Set
+
+from pydantic import UUID4, BaseModel, Field
+
+from recce.util.pydantic_model import pydantic_model_dump
+
+
+class RunType(Enum):
+    SIMPLE = "simple"
+    QUERY = "query"
+    QUERY_BASE = "query_base"
+    QUERY_DIFF = "query_diff"
+    VALUE_DIFF = "value_diff"
+    VALUE_DIFF_DETAIL = "value_diff_detail"
+    SCHEMA_DIFF = "schema_diff"
+    PROFILE = "profile"
+    PROFILE_DIFF = "profile_diff"
+    ROW_COUNT = "row_count"
+    ROW_COUNT_DIFF = "row_count_diff"
+    LINEAGE_DIFF = "lineage_diff"
+    TOP_K_DIFF = "top_k_diff"
+    HISTOGRAM_DIFF = "histogram_diff"
+
+    def __str__(self):
+        return self.value
+
+
+class RunProgress(BaseModel):
+    message: str
+    percentage: float
+
+
+class RunStatus(Enum):
+    FINISHED = "finished"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+    RUNNING = "running"
+
+
+class Run(BaseModel):
+    type: RunType
+    name: Optional[str] = None
+    params: Optional[dict] = None
+    check_id: Optional[UUID4] = None
+    result: Optional[dict] = None
+    error: Optional[str] = None
+    status: Optional[RunStatus] = None
+    progress: Optional[RunProgress] = None
+    run_id: UUID4 = Field(default_factory=uuid.uuid4)
+    run_at: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
+
+    def __init__(self, **data):
+        type = data.get("type")
+
+        if "result" in data and data["result"] is not None:
+            result = data.get("result")
+
+            if type in [RunType.QUERY.value, RunType.QUERY_BASE.value]:
+                from recce.tasks.query import QueryResult
+
+                data["result"] = pydantic_model_dump(QueryResult(**result))
+            elif type == RunType.QUERY_DIFF.value:
+                from recce.tasks.query import QueryDiffResult
+
+                data["result"] = pydantic_model_dump(QueryDiffResult(**result))
+            elif type == RunType.PROFILE.value:
+                from recce.tasks.profile import ProfileResult
+
+                data["result"] = pydantic_model_dump(ProfileResult(**result))
+            elif type == RunType.PROFILE_DIFF.value:
+                from recce.tasks.profile import ProfileDiffResult
+
+                data["result"] = pydantic_model_dump(ProfileDiffResult(**result))
+            elif type == RunType.VALUE_DIFF.value:
+                from recce.tasks.valuediff import ValueDiffResult
+
+                data["result"] = pydantic_model_dump(ValueDiffResult(**result))
+            elif type == RunType.VALUE_DIFF_DETAIL.value:
+                from recce.tasks.valuediff import ValueDiffDetailResult
+
+                data["result"] = pydantic_model_dump(ValueDiffDetailResult(**result))
+
+        super().__init__(**data)
+
+
+class Check(BaseModel):
+    name: str
+    description: Optional[str] = None
+    type: RunType
+    params: Optional[dict] = {}
+    view_options: Optional[dict] = {}
+    check_id: UUID4 = Field(default_factory=uuid.uuid4)
+    session_id: Optional[UUID4] = Field(default=None)
+    is_checked: bool = False
+    is_preset: bool = False
+    created_by: Optional[str] = None
+    updated_by: Optional[str] = None
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc).replace(microsecond=0))
+    updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc).replace(microsecond=0))
+
+    def merge(self, other) -> bool:
+        if not self.is_preset and not other.is_preset and self.check_id != other.check_id:
+            raise ValueError(f"check_id mismatch: {self.check_id} != {other.check_id}")
+
+        if self.updated_at and other.updated_at and self.updated_at >= other.updated_at:
+            return False
+
+        self.name = other.name
+        self.description = other.description
+        self.type = other.type
+        self.params = other.params
+        self.view_options = other.view_options
+        self.is_checked = other.is_checked
+        self.is_preset = other.is_preset
+        self.updated_at = other.updated_at
+        return True
+
+
+ChangeStatus = Literal[
+    "added",
+    "removed",
+    "modified",
+]
+ChangeCategory = Literal[
+    "breaking",
+    "non_breaking",
+    "partial_breaking",
+    "unknown",
+]
+
+
+class NodeChange(BaseModel):
+    category: ChangeCategory
+    columns: Optional[dict[str, ChangeStatus]] = None
+
+
+class NodeDiff(BaseModel):
+    change_status: ChangeStatus
+    change: Optional[NodeChange] = None  # Only available if change_status is 'modified'
+
+
+class LineageDiff(BaseModel):
+    base: dict
+    current: dict
+    diff: dict[str, NodeDiff]
+
+
+# Column Level Linage
+class CllColumnDep(BaseModel):
+    node: str
+    column: str
+
+
+class CllColumn(BaseModel):
+    id: Optional[str] = None
+    table_id: Optional[str] = None
+    name: Optional[str] = None
+
+    # data type
+    type: Optional[str] = None
+
+    # transformation type
+    transformation_type: Literal["source", "passthrough", "renamed", "derived", "unknown"] = "unknown"
+
+    # change analysis
+    change_status: Optional[ChangeStatus] = None
+
+    # column-to-column dependencies
+    depends_on: List[CllColumnDep] = Field(default_factory=list)
+
+
+class CllNode(BaseModel):
+    id: str
+    name: str
+    package_name: str
+    resource_type: str
+    raw_code: Optional[str] = None
+    source_name: Optional[str] = None
+
+    # change analysis
+    change_status: Optional[ChangeStatus] = None
+    change_category: Optional[ChangeCategory] = None
+
+    # Column to column dependencies
+    columns: Dict[str, CllColumn] = Field(default_factory=dict)
+
+    # If the node is impacted. Only used if option 'change_analysis' is set
+    impacted: Optional[bool] = None
+
+    @classmethod
+    def build_cll_node(cls, manifest, resource_key, node_id) -> Optional["CllNode"]:
+        resources = getattr(manifest, resource_key)
+        if node_id not in resources:
+            return None
+        n = resources[node_id]
+        if resource_key == "nodes" and n.resource_type not in ["model", "seed", "snapshot"]:
+            return None
+        cll_node = CllNode(
+            id=n.unique_id,
+            name=n.name,
+            package_name=n.package_name,
+            resource_type=n.resource_type,
+        )
+        if resource_key == "sources":
+            cll_node.source_name = n.source_name
+        elif resource_key == "nodes":
+            cll_node.raw_code = n.raw_code
+        return cll_node
+
+
+class CllData(BaseModel):
+    nodes: Dict[str, CllNode] = Field(default_factory=dict)
+    columns: Dict[str, CllColumn] = Field(default_factory=dict)
+    parent_map: Dict[str, Set[str]] = Field(default_factory=dict)
+    child_map: Dict[str, Set[str]] = Field(default_factory=dict)

package/recce-source/recce/pull_request.py

@@ -0,0 +1,124 @@
+import json
+import os
+from typing import Optional, Union
+
+import requests
+from pydantic import BaseModel
+
+from recce.git import hosting_repo
+from recce.github import recce_pr_information
+from recce.util.pydantic_model import pydantic_model_dump
+
+
+class PullRequestInfo(BaseModel):
+    id: Optional[Union[int, str]] = None
+    title: Optional[str] = None
+    url: Optional[str] = None
+    branch: Optional[str] = None
+    base_branch: Optional[str] = None
+    repository: Optional[str] = None
+
+    def to_dict(self):
+        return pydantic_model_dump(self)
+
+
+def fetch_pr_metadata(**kwargs):
+    pr_info = PullRequestInfo()
+
+    # fetch from github action event path
+    metadata = fetch_pr_metadata_from_event_path()
+    if metadata is not None:
+        pr_info.id = metadata.get("github_pr_id")
+        pr_info.url = metadata.get("github_pr_url")
+        pr_info.title = metadata.get("github_pr_title")
+        pr_info.repository = metadata.get("github_repository")
+    else:
+        repo = hosting_repo()
+        pr, message = recce_pr_information(github_token=kwargs.get("github_token"))
+        if kwargs.get("cloud") and message:
+            print(message)
+
+        if pr:
+            pr_info.repository = repo
+            pr_info.id = pr.number
+            pr_info.title = pr.title
+            pr_info.url = pr.html_url
+            pr_info.base_branch = pr.base.ref
+            pr_info.branch = pr.head.ref
+
+    # fetch from cli arguments
+    if pr_info.url is None and "github_pull_request_url" in kwargs:
+        pr_info.url = kwargs.get("github_pull_request_url")
+
+    if pr_info.branch is None:
+        pr_info.branch = kwargs.get("git_current_branch")
+
+    if pr_info.base_branch is None:
+        pr_info.base_branch = kwargs.get("git_base_branch")
+
+    # fetch from env
+    if pr_info.url is None:
+        pr_info.url = os.getenv("RECCE_PR_URL")
+
+    return pr_info
+
+
+def fetch_pr_metadata_from_event_path() -> Optional[dict]:
+    """
+    If recce run is running in a GitHub Action, this function will return the pull request metadata.
+
+    Example:
+    {
+        "github_pr_id": 1,
+        "github_pr_url": "https://github.com/xyz/abc/pull/1,
+        "github_pr_title": "Update README.md",
+        "github_repository": "xyz/abc"
+    }
+
+    :return: dict
+    """
+
+    # get the event json from the path in GITHUB_EVENT_PATH
+    event_path = os.getenv("GITHUB_EVENT_PATH")
+    github_repository = os.getenv("GITHUB_REPOSITORY")
+    if event_path:
+        try:
+            with open(event_path, "r", encoding="utf-8") as event_file:
+                event_data = json.load(event_file)
+
+            pr_id = event_data["number"]
+            if event_data.get("pull_request"):
+                pull_request_data = event_data["pull_request"]
+                pr_url = pull_request_data["_links"]["html"]["href"]
+                pr_api = pull_request_data["_links"]["self"]["href"]
+                pr_title = _fetch_pr_title(pr_api)
+                return dict(
+                    github_pr_id=pr_id,
+                    github_pr_url=pr_url,
+                    github_pr_title=pr_title,
+                    github_repository=github_repository,
+                )
+            else:
+                print("Not a pull request event, skip.")
+        except Exception as e:
+            print("Cannot parse github action event", e)
+    return None
+
+
+def _fetch_pr_title(endpoint) -> Optional[str]:
+    github_token = os.getenv("GITHUB_TOKEN")
+
+    if github_token is None:
+        return None
+
+    try:
+        headers = {"Authorization": f"Bearer {github_token}"}
+        response = requests.get(endpoint, headers=headers)
+
+        if response.status_code == 200:
+            pull_request_data = response.json()
+            return pull_request_data.get("title")
+    except Exception as e:
+        print("Cannot fetch PR title: ", e)
+
+    return None