@datarecce/ui 0.1.30 → 0.1.31

package/recce-source/recce/summary.py

@@ -0,0 +1,575 @@
+import os
+import sys
+from typing import Dict, List, Optional, Set, Type, Union
+from uuid import UUID
+
+from pydantic import BaseModel
+
+from recce.apis.check_func import get_node_name_by_id
+from recce.core import RecceContext
+from recce.models import CheckDAO, Run, RunDAO, RunType
+from recce.tasks.core import TaskResultDiffer
+from recce.tasks.histogram import HistogramDiffTaskResultDiffer
+from recce.tasks.profile import ProfileDiffResultDiffer
+from recce.tasks.query import QueryDiffResultDiffer
+from recce.tasks.rowcount import RowCountDiffResultDiffer
+from recce.tasks.schema import SchemaDiffResultDiffer
+from recce.tasks.top_k import TopKDiffTaskResultDiffer
+from recce.tasks.valuediff import (
+    ValueDiffDetailTaskResultDiffer,
+    ValueDiffTaskResultDiffer,
+)
+
+RECCE_CLOUD_HOST = os.environ.get("RECCE_CLOUD_HOST", "https://cloud.datarecce.io")
+
+ADD_COLOR = "#1dce00"
+MODIFIED_COLOR = "#ffa502"
+REMOVE_COLOR = "#ff067e"
+
+MAX_MERMAID_TEXT_SIZE = 50000  # source: https://mermaid.js.org/config/schema-docs/config.html#maxtextsize
+
+
+def _warn(msg):
+    # print to stderr
+    print(f"warning: {msg}", file=sys.stderr)
+
+
+class Node:
+    id: str
+    name: str
+    data_from: str
+    resource_type: str
+    package_name: str
+    parents: List[str]
+    children: List[str]
+    base_data: dict
+    current_data: dict
+
+    def __init__(self, node_id: str, node_data: dict, data_from: str = "base"):
+        self.id = node_id
+        self.name = node_data["name"]
+        self.data_from = data_from
+        self.resource_type = node_data["resource_type"]
+        self.package_name = node_data["package_name"]
+        self.children = []
+        self.parents = []
+
+        self.base_data = {}
+        self.current_data = {}
+
+        if data_from == "base":
+            self.base_data = node_data
+        elif data_from == "current":
+            self.current_data = node_data
+
+    @property
+    def change_status(self):
+        base_checksum = self.base_data.get("checksum", {}).get("checksum")
+        curr_checksum = self.current_data.get("checksum", {}).get("checksum")
+        if self.data_from == "base":
+            return "removed"
+        elif self.data_from == "current":
+            return "added"
+        elif base_checksum and curr_checksum and base_checksum != curr_checksum:
+            return "modified"
+        return None
+
+    def update_data(self, node_data: dict, data_from: str):
+        if data_from not in ["base", "current"]:
+            raise ValueError(f"Invalid data_from value: {data_from}")
+        if self.data_from != data_from:
+            self.data_from = "both"
+
+            if data_from == "base":
+                self.base_data = node_data
+            elif data_from == "current":
+                self.current_data = node_data
+
+    def append_parent(self, parent_id: str):
+        if parent_id not in self.parents:
+            self.parents.append(parent_id)
+
+    def append_child(self, child_id: str):
+        if child_id not in self.children:
+            self.children.append(child_id)
+
+    def _cal_row_count_delta_percentage(self):
+        row_count_diff, run_result = _get_node_row_count_diff(self.id, self.name)
+        if row_count_diff:
+            base = run_result.get("base", 0)
+            current = run_result.get("curr", 0)
+            if int(current) > int(base):
+                p = (int(current) - int(base)) / int(current) * 100
+                return f'🔼 +{round(p, 2) if p > 0.1 else "<0.1"}%'
+            else:
+                p = (int(base) - int(current)) / int(current) * 100
+                return f'🔽 -{round(p, 2) if p > 0.1 else "<0.1"}%'
+        return None
+
+    def _get_schema_diff(self):
+        base_schema = self.base_data.get("columns", {})
+        current_schema = self.current_data.get("columns", {})
+        schema_diff = TaskResultDiffer.diff(base_schema, current_schema)
+        return schema_diff
+
+    def _what_changed(self, checks=None):
+        changes = []
+        if self.change_status == "added":
+            return ["Added Node"]
+        elif self.change_status == "removed":
+            return ["Removed Node"]
+        elif self.change_status == "modified":
+            changes.append("Code")
+        row_count_delta_percentage = self._cal_row_count_delta_percentage()
+        if row_count_delta_percentage:
+            changes.append(f"Row Count {row_count_delta_percentage}")
+        schema_diff = self._get_schema_diff()
+        if schema_diff:
+            changes.append("Schema")
+
+        if checks:
+            for check in checks:
+                check_type = check.type
+                if check_type == RunType.ROW_COUNT_DIFF or check_type == RunType.SCHEMA_DIFF:
+                    # Skip the row count and schema diff check, since we already have it.
+                    continue
+                if check.node_ids and self.id in check.node_ids:
+                    changes.append(str(check.type).replace("_", " ").title())
+        return changes
+
+    def get_node_str(self, checks=None):
+        is_changed = False
+        style = None
+
+        if self.change_status is not None:
+            is_changed = True
+            if self.change_status == "added":
+                style = f"style {self.id} stroke:{ADD_COLOR}"
+            elif self.change_status == "modified":
+                style = f"style {self.id} stroke:{MODIFIED_COLOR}"
+            elif self.change_status == "removed":
+                style = f"style {self.id} stroke:{REMOVE_COLOR}"
+
+        if checks:
+            for check in checks:
+                if check.node_ids and self.id in check.node_ids:
+                    is_changed = True
+
+        content_output = f'{self.id}["{self.name}'
+        if is_changed:
+            content_output += "\n\n[What's Changed]\n"
+            changes = self._what_changed(checks)
+            content_output += ", ".join(changes)
+
+        content_output += '"]\n'
+        if style:
+            content_output += f"{style}\n"
+        return content_output
+
+
+class Edge:
+    id: str
+    edge_from: str
+    child_id: str
+    parent_id: str
+    change_status: Union[str, None]
+
+    def __init__(self, edge_id: str, parent_id: str, child_id: str, edge_from: str = "base"):
+        self.id = edge_id
+        self.edge_from = edge_from
+        self.child_id = child_id
+        self.parent_id = parent_id
+
+    def update_edge_from(self, edge_from: str):
+        if self.edge_from != edge_from:
+            self.edge_from = "both"
+
+
+class CheckSummary(BaseModel):
+    id: UUID
+    type: RunType
+    name: str
+    description: str
+    changes: dict
+    node_ids: Optional[List[str]]
+    changed_nodes: Optional[List[str]]
+
+    @property
+    def related_nodes(self):
+        nodes = []
+
+        for node_id in self.node_ids or []:
+            name = get_node_name_by_id(node_id)
+            nodes.append(name)
+
+        return nodes
+
+
+check_result_differ_registry: Dict[RunType, Type[TaskResultDiffer]] = {
+    RunType.VALUE_DIFF: ValueDiffTaskResultDiffer,
+    RunType.VALUE_DIFF_DETAIL: ValueDiffDetailTaskResultDiffer,
+    RunType.ROW_COUNT_DIFF: RowCountDiffResultDiffer,
+    RunType.QUERY_DIFF: QueryDiffResultDiffer,
+    RunType.TOP_K_DIFF: TopKDiffTaskResultDiffer,
+    RunType.HISTOGRAM_DIFF: HistogramDiffTaskResultDiffer,
+    RunType.PROFILE_DIFF: ProfileDiffResultDiffer,
+}
+
+
+def differ_factory(run: Run):
+    differ_clz = check_result_differ_registry.get(run.type)
+    if not differ_clz:
+        raise NotImplementedError()
+    return differ_clz(run)
+
+
+class LineageGraph:
+    nodes: Dict[str, Node] = {}
+    edges: Dict[str, Edge] = {}
+    checks: List[CheckSummary] = None
+
+    def create_node(self, node_id: str, node_data: dict, data_from: str = "base"):
+        if node_id not in self.nodes:
+            self.nodes[node_id] = Node(node_id, node_data, data_from)
+        else:
+            self.nodes[node_id].update_data(node_data, data_from)
+
+    def create_edge(self, parent_id: str, child_id: str, edge_from: str = "base"):
+        if parent_id not in self.nodes:
+            _warn(f"Parent node {parent_id} not found in graph")
+            return
+        if child_id not in self.nodes:
+            _warn(f"Child node {child_id} not found in graph")
+            return
+
+        edge_id = f"{parent_id}-->{child_id}"
+        if edge_id in self.edges:
+            self.edges[edge_id].update_edge_from(edge_from)
+        else:
+            self.edges[edge_id] = Edge(edge_id, parent_id, child_id, edge_from)
+            self.nodes[parent_id].append_child(child_id)
+            self.nodes[child_id].append_parent(parent_id)
+
+    @property
+    def modified_set(self) -> Set[str]:
+        return set(
+            [node_id for node_id, node in self.nodes.items() if node.change_status in ["added", "removed", "modified"]]
+        )
+
+    def get_edge_str(self, edge_id):
+        edge = self.edges[edge_id]
+        child = self.nodes[edge.child_id]
+
+        if child.change_status == "removed":
+            return f"{edge.parent_id}-.->{edge.child_id}\n"
+        if child.change_status is None or child.change_status == "modified":
+            return f"{edge.parent_id}---->{edge.child_id}\n"
+        if child.change_status == "added":
+            return f"{edge.parent_id}-...->{edge.child_id}\n"
+
+
+def _build_lineage_graph(base, current) -> LineageGraph:
+    graph = LineageGraph()
+
+    # Get the current package name to filter nodes (from the current manifest metadata)
+    package_name = None
+    manifest_metadata = current.get("manifest_metadata")
+    if manifest_metadata and hasattr(manifest_metadata, "project_name"):
+        # The default package name is the project name
+        package_name = manifest_metadata.project_name
+
+    # Init Graph nodes with base & current nodes
+    for node_id, node_data in base.get("nodes", {}).items():
+        # Skip nodes that are not from the current package
+        if package_name and node_data.get("package_name") != package_name:
+            continue
+        graph.create_node(node_id, node_data, "base")
+
+    for node_id, node_data in current.get("nodes", {}).items():
+        # Skip nodes that are not from the current package
+        if package_name and node_data.get("package_name") != package_name:
+            continue
+        if node_id not in graph.nodes:
+            node = Node(node_id, node_data, "current")
+            graph.nodes[node_id] = node
+        else:
+            node = graph.nodes[node_id]
+            node.update_data(node_data, "current")
+
+    # Build edges
+    for child_id, parents in base.get("parent_map", {}).items():
+        for parent_id in parents:
+            if child_id not in graph.nodes or parent_id not in graph.nodes:
+                continue
+
+            graph.create_edge(parent_id, child_id, "base")
+    for child_id, parents in current.get("parent_map", {}).items():
+        for parent_id in parents:
+            if child_id not in graph.nodes or parent_id not in graph.nodes:
+                continue
+
+            graph.create_edge(parent_id, child_id, "current")
+
+    return graph
+
+
+def _build_node_schema(lineage, node_id):
+    return lineage.get("nodes", {}).get(node_id, {}).get("columns", {})
+
+
+def _get_node_row_count_diff(node_id, node_name):
+    row_count_runs = RunDAO().list(type_filter=RunType.ROW_COUNT_DIFF)
+    for run in row_count_runs:
+        node_ids = (run.params or {}).get("node_ids") or []
+        if node_id in node_ids:
+            result = run.result.get(node_name, {})
+            diff = TaskResultDiffer.diff(result.get("base"), result.get("curr"))
+            return diff, result
+        elif run.params.get("node_id") == node_id:
+            result = run.result.get(node_name, {})
+            diff = TaskResultDiffer.diff(result.get("base"), result.get("curr"))
+            return diff, result
+    return None, None
+
+
+def _generate_mismatched_nodes_summary(check: CheckSummary, limit: int = 3) -> str:
+    if not check.related_nodes:
+        return "N/A"
+
+    nodes = check.related_nodes
+    if check.changed_nodes:
+        # the mismatch nodes within the related nodes (when apply node selection)
+        # currently only schema_diff and row_count_diff are supported to apply node selection
+        nodes = check.changed_nodes
+
+    if len(nodes) <= limit:
+        return ", ".join(nodes)
+
+    display_nodes = nodes[: limit - 1]
+    return ", ".join(display_nodes) + f", and {len(nodes) - len(display_nodes)} more nodes"
+
+
+def generate_summary_metadata(base_lineage, curr_lineage):
+    from py_markdown_table.markdown_table import markdown_table
+
+    base_manifest = base_lineage.get("manifest_metadata")
+    base_catalog = base_lineage.get("catalog_metadata")
+    curr_manifest = curr_lineage.get("manifest_metadata")
+    curr_catalog = curr_lineage.get("catalog_metadata")
+
+    metadata = [
+        {
+            "": "Base",
+            "Manifest": base_manifest.generated_at.strftime("%Y-%m-%d %H:%M:%S"),
+            "Catalog": base_catalog.generated_at.strftime("%Y-%m-%d %H:%M:%S") if base_catalog else "N/A",
+        },
+        {
+            "": "Current",
+            "Manifest": curr_manifest.generated_at.strftime("%Y-%m-%d %H:%M:%S"),
+            "Catalog": curr_catalog.generated_at.strftime("%Y-%m-%d %H:%M:%S") if curr_catalog else "N/A",
+        },
+    ]
+
+    return (
+        markdown_table(metadata)
+        .set_params(
+            quote=False,
+            row_sep="markdown",
+            padding_width=1,
+            padding_weight="right",  # Aligns the cell's contents to the beginning of the cell
+        )
+        .get_markdown()
+    )
+
+
+def generate_check_summary(base_lineage, curr_lineage) -> (List[CheckSummary], Dict[str, int]):
+    runs = RunDAO().list()
+    checks = CheckDAO().list()
+    checks_summary: List[CheckSummary] = []
+    failed_checks_count = 0
+
+    # TODO: find a way to count failed checks, currently the state file won't include failed checks
+
+    def _find_run_by_check_id(check_id):
+        runs_for_check = [r for r in runs if str(check_id) == str(r.check_id)]
+        if runs_for_check:
+            return runs_for_check[-1]
+        return None
+
+    for check in checks:
+        run = _find_run_by_check_id(check.check_id)
+        differ = None
+        if run is not None and run.error is not None:
+            failed_checks_count += 1
+            continue
+        elif check.type == RunType.SCHEMA_DIFF:
+            differ = SchemaDiffResultDiffer(check, base_lineage, curr_lineage)
+        elif (
+            check.type
+            in [
+                RunType.ROW_COUNT_DIFF,
+                RunType.QUERY_DIFF,
+                RunType.VALUE_DIFF,
+                RunType.VALUE_DIFF_DETAIL,
+                RunType.PROFILE_DIFF,
+                RunType.TOP_K_DIFF,
+                RunType.HISTOGRAM_DIFF,
+            ]
+            and run is not None
+        ):
+            # Check the result is changed or not
+            differ = differ_factory(run)
+
+        if differ and differ.changes is not None:
+            checks_summary.append(
+                CheckSummary(
+                    id=check.check_id,
+                    type=check.type,
+                    name=check.name,
+                    description=check.description,
+                    changes=differ.changes,
+                    node_ids=differ.related_node_ids,
+                    changed_nodes=differ.changed_nodes,
+                )
+            )
+
+    return checks_summary, {
+        "total": len(checks),
+        "mismatch": len(checks_summary),
+        "failed": failed_checks_count,
+    }
+
+
+def generate_mermaid_lineage_graph(graph: LineageGraph):
+    content = up_to_level_content = "graph LR\n"
+    is_not_modified = False
+    # Only show the modified nodes and there children
+    queue = list(graph.modified_set)
+    display_nodes = set()
+    display_edge = set()
+
+    if len(queue) == 0:
+        is_not_modified = True
+
+    while len(queue) > 0:
+
+        level_nodes = len(queue)
+        for _ in range(level_nodes):
+            node_id = queue.pop(0)
+            if node_id in display_nodes:
+                # Skip if already displayed
+                continue
+
+            display_nodes.add(node_id)
+            node = graph.nodes[node_id]
+            content += node.get_node_str(graph.checks)
+            for child_id in node.children:
+                queue.append(child_id)
+                edge_id = f"{node_id}-->{child_id}"
+                if edge_id not in display_edge:
+                    display_edge.add(edge_id)
+                    content += graph.get_edge_str(edge_id)
+
+        if len(content) > MAX_MERMAID_TEXT_SIZE:
+            break
+
+        up_to_level_content = content
+
+    return up_to_level_content, is_not_modified, len(content) > MAX_MERMAID_TEXT_SIZE
+
+
+def generate_markdown_summary(ctx: RecceContext, summary_format: str = "markdown"):
+    lineage_diff = ctx.get_lineage_diff()
+    summary_metadata = generate_summary_metadata(lineage_diff.base, lineage_diff.current)
+    graph = _build_lineage_graph(lineage_diff.base, lineage_diff.current)
+    graph.checks, check_statistics = generate_check_summary(lineage_diff.base, lineage_diff.current)
+    mermaid_content, is_empty_graph, is_partial_graph = generate_mermaid_lineage_graph(graph)
+    check_content = generate_check_content(graph, check_statistics)
+
+    if summary_format == "mermaid":
+        return mermaid_content
+    elif summary_format == "check":
+        return check_content
+    elif summary_format == "markdown":
+
+        content = "# Recce Summary\n"
+        content += f"## Manifest Information\n{summary_metadata}\n"
+
+        if is_empty_graph is False:
+            content += f"""
+## Lineage Graph
+{"_Too many nodes to generate! Please see the full lineage graph on Recce instance._" if is_partial_graph else ''}
+```mermaid
+{mermaid_content}
+```
+"""
+        else:
+            content += """
+## Lineage Graph
+No changed module was detected.
+"""
+        if check_content:
+            content += check_content
+
+        if ctx.state_loader.cloud_mode and ctx.state_loader.pr_info is not None:
+            pr_info = ctx.state_loader.pr_info
+            if pr_info.repository is not None and pr_info.id is not None:
+                # the classic route will be deprecated soon
+                content += f"\nSee PR page: {RECCE_CLOUD_HOST}/classic/{pr_info.repository}/pulls/{pr_info.id}\n"
+
+        return content
+
+
+def generate_check_content(graph, check_statistics):
+    from py_markdown_table.markdown_table import markdown_table
+
+    content = ""
+    check_content = None
+    # Generate the check summary if we found any changes
+    if len(graph.checks) > 0:
+        data = []
+        for check in graph.checks:
+            data.append(
+                {
+                    "Name": check.name,
+                    "Type": str(check.type).replace("_", " ").title(),
+                    "Mismatched Nodes": _generate_mismatched_nodes_summary(check),
+                    # Temporarily remove the type of changes, until we implement a better way to display it.
+                    # 'Type of Changes': _formate_changes(check.changes)
+                }
+            )
+        check_content = (
+            markdown_table(data)
+            .set_params(
+                quote=False,
+                row_sep="markdown",
+                padding_width=1,
+                padding_weight="right",  # Aligns the cell's contents to the beginning of the cell
+            )
+            .get_markdown()
+        )
+
+    if check_statistics.get("total", 0) > 0:
+        warning_message = ""
+        statistics = {
+            "Checks Run": check_statistics.get("total", 0),
+            "Data Mismatch Detected": check_statistics.get("mismatch", 0),
+        }
+        if check_statistics.get("failed", 0) > 0:
+            statistics["Incomplete Checks"] = check_statistics.get("failed", 0)
+            warning_message = """
+:warning: **Incomplete Checks** refers to checks that did not successfully run due to configuration or SQL errors.
+Please check the output of `recce run` for more information
+"""
+        check_summary = markdown_table([statistics]).set_params(quote=False, row_sep="markdown").get_markdown()
+        content += f"""
+## Checks Summary
+{check_summary}
+{warning_message}
+"""
+    if check_content:
+        content += f"""
+### Checks of Data Mismatch Detected
+{check_content}
+"""
+    return content

package/recce-source/recce/tasks/__init__.py

@@ -0,0 +1,23 @@
+from .core import Task
+from .histogram import HistogramDiffTask
+from .profile import ProfileDiffTask, ProfileTask
+from .query import QueryBaseTask, QueryDiffTask, QueryTask
+from .rowcount import RowCountDiffTask, RowCountTask
+from .top_k import TopKDiffTask
+from .valuediff import ValueDiffDetailTask, ValueDiffTask
+
+# Explicitly declare exports
+__all__ = [
+    "Task",
+    "HistogramDiffTask",
+    "ProfileDiffTask",
+    "ProfileTask",
+    "QueryBaseTask",
+    "QueryDiffTask",
+    "QueryTask",
+    "RowCountDiffTask",
+    "RowCountTask",
+    "TopKDiffTask",
+    "ValueDiffDetailTask",
+    "ValueDiffTask",
+]