@datarecce/ui 0.1.30 → 0.1.32

package/recce-source/recce/config.py

@@ -0,0 +1,127 @@
+# Configuration file for recce
+from rich.console import Console
+
+from recce import yaml
+from recce.exceptions import RecceConfigException
+from recce.util import SingletonMeta
+
+RECCE_CONFIG_FILE = "recce.yml"
+RECCE_PRESET_CHECK_COMMENT = """Preset Checks
+Please see https://docs.datarecce.io/features/preset-checks/
+"""
+RECCE_ERROR_LOG_FILE = "recce_error.log"
+console = Console()
+
+
+class RecceConfig(metaclass=SingletonMeta):
+    def __init__(self, config_file=RECCE_CONFIG_FILE):
+        self.config_file = config_file
+        self.config = None
+        self.load()
+
+    def load(self):
+        try:
+            with open(self.config_file, "r", encoding="utf-8") as f:
+                config = yaml.safe_load(f)
+                self.config = config if config else {}
+            self._verify_preset_checks()
+        except FileNotFoundError:
+            console.print(f"[[orange3]NOTICE[/orange3]] Generate default Recce config file at '{self.config_file}'")
+            self.config = self.generate_template()
+            self.save()
+
+    def _verify_preset_checks(self):
+        from recce.tasks.core import CheckValidator
+
+        if not self.config.get("checks"):
+            return
+
+        for check in self.config["checks"]:
+            try:
+                check_type = check.get("type")
+                if check_type is None:
+                    raise ValueError(f'Check type is required for check "{check}"')
+                if check_type == "lineage_diff":
+                    from recce.tasks.lineage import LineageDiffCheckValidator
+
+                    validator = LineageDiffCheckValidator()
+                elif check_type == "schema_diff":
+                    from recce.tasks.schema import SchemaDiffCheckValidator
+
+                    validator = SchemaDiffCheckValidator()
+                elif check_type == "row_count_diff":
+                    from recce.tasks.rowcount import RowCountDiffCheckValidator
+
+                    validator = RowCountDiffCheckValidator()
+                elif check_type == "query":
+                    from recce.tasks.query import QueryCheckValidator
+
+                    validator = QueryCheckValidator()
+                elif check_type == "query_diff":
+                    from recce.tasks.query import QueryDiffCheckValidator
+
+                    validator = QueryDiffCheckValidator()
+                elif check_type == "value_diff" or check_type == "value_diff_detail":
+                    from recce.tasks.valuediff import ValueDiffCheckValidator
+
+                    validator = ValueDiffCheckValidator()
+                elif check_type == "profile_diff":
+                    from recce.tasks.profile import ProfileCheckValidator
+
+                    validator = ProfileCheckValidator()
+                elif check_type == "top_k_diff":
+                    from recce.tasks.top_k import TopKDiffCheckValidator
+
+                    validator = TopKDiffCheckValidator()
+                elif check_type == "histogram_diff":
+                    from recce.tasks.histogram import HistogramDiffCheckValidator
+
+                    validator = HistogramDiffCheckValidator()
+                else:
+                    validator = CheckValidator()
+                validator.validate(check)
+            except Exception as e:
+                import json
+
+                raise RecceConfigException(
+                    f"Load preset checks failed from '{self.config_file}'\n{json.dumps(check, indent=2)}", cause=e
+                )
+
+    def generate_template(self):
+        data = yaml.CommentedMap(checks=yaml.CommentedSeq())
+        data.yaml_set_comment_before_after_key("checks", before=RECCE_PRESET_CHECK_COMMENT)
+        # Define default preset checks
+        default_checks = [
+            yaml.CommentedMap(
+                name="Row count diff",
+                description="Check the row count diff for all table models.",
+                type="row_count_diff",
+                params={"select": "state:modified,config.materialized:table"},
+            ),
+            yaml.CommentedMap(
+                name="Schema diff",
+                description="Check the schema diff for all nodes.",
+                type="schema_diff",
+            ),
+        ]
+
+        for check in default_checks:
+            data["checks"].append(check)
+
+        return data
+
+    def get(self, key, default=None):
+        return self.config.get(key, default)
+
+    def set(self, key, value):
+        self.config[key] = value
+
+    def save(self):
+        with open(RECCE_CONFIG_FILE, "w", encoding="utf-8") as f:
+            yaml.dump(self.config, f)
+
+    def __str__(self):
+        return yaml.dump(self.config)
+
+    def __repr__(self):
+        return self.__str__()

package/recce-source/recce/connect_to_cloud.py

@@ -0,0 +1,138 @@
+import base64
+import os.path
+import random
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+from typing import Tuple
+from urllib.parse import parse_qs, urlparse
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import padding, rsa
+from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey
+from rich.console import Console
+
+from recce.event import update_recce_api_token
+from recce.exceptions import RecceConfigException
+from recce.util.onboarding_state import update_onboarding_state
+from recce.util.recce_cloud import RECCE_CLOUD_BASE_URL, RecceCloud
+
+console = Console()
+
+static_folder_path = Path(__file__).parent / "data"
+_server_lock = threading.Lock()
+_connection_url = None
+
+
+def decrypt_code(private_key: RSAPrivateKey, code: str) -> str:
+    ciphertext = base64.b64decode(code)
+    plaintext = private_key.decrypt(
+        ciphertext,
+        padding.OAEP(
+            mgf=padding.MGF1(algorithm=hashes.SHA1()),  # Node.js uses SHA1 by default
+            algorithm=hashes.SHA1(),
+            label=None,
+        ),
+    )
+    return plaintext.decode("utf-8")
+
+
+def handle_callback_request(query_string: str, private_key: RSAPrivateKey):
+    query_params = parse_qs(query_string)
+    code = query_params.get("code", [None])[0]
+    if not code:
+        raise RecceConfigException("Missing `code` in query")
+
+    api_token = decrypt_code(private_key, code)
+    if not RecceCloud(api_token).verify_token():
+        raise RecceConfigException("Invalid Recce Cloud API token")
+
+    update_recce_api_token(api_token)
+    update_onboarding_state(api_token, False)
+
+    return api_token  # for testability/debugging
+
+
+def make_callback_handler(private_key: RSAPrivateKey):
+    class OneTimeHTTPRequestHandler(BaseHTTPRequestHandler):
+        def do_GET(self):
+            try:
+                with open(os.path.join(static_folder_path, "auth_callback.html"), "r", encoding="utf-8") as f:
+                    callback_html_content = f.read()
+
+                # Parse query parameters
+                parsed_url = urlparse(self.path)
+
+                handle_callback_request(parsed_url.query, private_key)
+
+                # Construct HTML content
+                self.send_response(200)
+                self.send_header("Content-Type", "text/html")
+                self.send_header("Content-Length", str(len(callback_html_content.encode())))
+                self.end_headers()
+                self.wfile.write(callback_html_content.encode())
+
+            except Exception:
+                console.print_exception()
+                self.send_response(500)
+                self.end_headers()
+                self.wfile.write(b"<h1>Internal Server Error</h1>")
+            finally:
+                # Shut down the server after handling the first request
+                # Shutdown in a new thread to avoid deadlock
+                self.server.server_close()
+                threading.Thread(target=self.server.shutdown, daemon=True).start()
+
+        def log_message(self, format, *args):
+            # Suppress default logging
+            return
+
+    return OneTimeHTTPRequestHandler
+
+
+def is_callback_server_running():
+    return _server_lock.locked()
+
+
+def get_connection_url():
+    return _connection_url
+
+
+def run_one_time_http_server(private_key: RSAPrivateKey, port=8080):
+    handler = make_callback_handler(private_key)
+    server = HTTPServer(("localhost", port), handler)
+    server.serve_forever()
+
+
+def prepare_connection_url(public_key: RSAPublicKey):
+    public_key_pem_bytes = public_key.public_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
+    )
+    public_key_pem_str = base64.b64encode(public_key_pem_bytes).decode("utf-8")
+    callback_port = random.randint(10000, 15000)
+    connect_url = f"{RECCE_CLOUD_BASE_URL}/connect?_key={public_key_pem_str}&_port={callback_port}"
+    return connect_url, callback_port
+
+
+def generate_key_pair() -> Tuple[RSAPrivateKey, RSAPublicKey]:
+    key_size = 2048  # Should be at least 2048
+
+    private_key = rsa.generate_private_key(
+        public_exponent=65537, key_size=key_size, backend=default_backend()  # Do not change
+    )
+
+    public_key = private_key.public_key()
+    return private_key, public_key
+
+
+def connect_to_cloud_background_task(private_key: RSAPrivateKey, callback_port, connection_url):
+    if is_callback_server_running():
+        return
+
+    with _server_lock:
+        global _connection_url
+        _connection_url = connection_url
+        run_one_time_http_server(private_key, callback_port)
+        _connection_url = None

package/recce-source/recce/core.py

@@ -0,0 +1,334 @@
+import hashlib
+import json
+import logging
+import os
+from dataclasses import dataclass, field
+from typing import Callable, Dict, List, Optional, Set, Tuple
+
+from recce.adapter.base import BaseAdapter
+from recce.models import Check, Run
+from recce.models.types import LineageDiff
+from recce.state import (
+    GitRepoInfo,
+    PullRequestInfo,
+    RecceState,
+    RecceStateLoader,
+    RecceStateMetadata,
+)
+from recce.util.recce_cloud import set_recce_cloud_onboarding_state
+
+logger = logging.getLogger("uvicorn")
+
+
+@dataclass
+class RecceContext:
+    review_mode: bool = False
+    adapter_type: str = None
+    adapter: BaseAdapter = None
+    state_loader: RecceStateLoader = None
+    runs: List[Run] = field(default_factory=list)
+    checks: List[Check] = field(default_factory=list)
+
+    @classmethod
+    def load(cls, **kwargs):
+        state_loader: RecceStateLoader = kwargs.get("state_loader")
+        is_review_mode = kwargs.get("review", False)
+
+        context = cls(
+            review_mode=is_review_mode,
+            state_loader=state_loader,
+        )
+
+        # Initiate the adapter
+        if kwargs.get("sqlmesh", False):
+            logger.warning("SQLMesh adapter is still in EXPERIMENTAL mode.")
+            from recce.adapter.sqlmesh_adapter import SqlmeshAdapter
+
+            context.adapter_type = "sqlmesh"
+            context.adapter = SqlmeshAdapter.load(**kwargs)
+        else:
+            from recce.adapter.dbt_adapter import DbtAdapter
+
+            context.adapter_type = "dbt"
+            context.adapter = DbtAdapter.load(**kwargs)
+
+        # Import state
+        if state_loader is not None:
+            state = state_loader.load()
+            if state:
+                context.import_state(state)
+
+            if is_review_mode:
+                if not state:
+                    raise Exception("The state file is required for review mode")
+
+        return context
+
+    def get_model(self, model_id: str, base=False):
+        return self.adapter.get_model(model_id, base=base)
+
+    def get_node_name_by_id(self, unique_id: object) -> object:
+        return self.adapter.get_node_name_by_id(unique_id)
+
+    def generate_sql(self, sql_template: str, base: bool = False, context: Dict = {}):
+        self.adapter.generate_sql(sql_template, base=base, context=context)
+
+    def get_lineage(self, base: Optional[bool] = False):
+        return self.adapter.get_lineage(base=base)
+
+    def get_lineage_diff(self) -> LineageDiff:
+        return self.adapter.get_lineage_diff()
+
+    def build_name_to_unique_id_index(self, excluded_types: Set = None) -> Dict[str, str]:
+        name_to_unique_id = {}
+        curr = self.get_lineage(base=False)
+        base = self.get_lineage(base=True)
+
+        for unique_id, node in curr["nodes"].items():
+            if excluded_types and node.get("resource_type") in excluded_types:
+                continue
+            name_to_unique_id[node["name"]] = unique_id
+        for unique_id, node in base["nodes"].items():
+            if excluded_types and node.get("resource_type") in excluded_types:
+                continue
+            name_to_unique_id[node["name"]] = unique_id
+        return name_to_unique_id
+
+    def start_monitor_artifacts(self, callback: Callable = None):
+        self.adapter.start_monitor_artifacts(callback=callback)
+
+    def stop_monitor_artifacts(self):
+        self.adapter.stop_monitor_artifacts()
+
+    def refresh_manifest(self, refresh_file_path: str = None):
+        self.adapter.refresh(refresh_file_path)
+
+    def start_monitor_base_env(self, callback: Callable = None):
+        self.adapter.start_monitor_base_env(callback=callback)
+
+    def stop_monitor_base_env(self):
+        self.adapter.stop_monitor_base_env()
+
+    def export_state(self) -> RecceState:
+        """
+        Export the state to a RecceState object.
+        """
+        state = RecceState()
+        state.metadata = RecceStateMetadata()
+
+        # runs & checks & artifacts
+        state.runs = self.runs
+        state.checks = self.checks
+        state.artifacts = self.adapter.export_artifacts()
+
+        # git & pull_request. If in review mode, use the review state
+        if self.review_mode:
+            state.git = self.state_loader.state.git
+            state.pull_request = self.state_loader.state.pull_request
+        else:
+            git = GitRepoInfo.from_current_repository()
+            if git:
+                state.git = git
+            if self.state_loader.pr_info:
+                state.pull_request = self.state_loader.pr_info
+
+        return state
+
+    def export_demo_state(self) -> RecceState:
+        """
+        Export the demo state to a RecceState object for the demo sites.
+        """
+        state = RecceState()
+        state.metadata = RecceStateMetadata()
+
+        # runs & checks
+        state.runs = self.runs
+        state.checks = self.checks
+        state.artifacts = self.adapter.export_artifacts()
+        git = GitRepoInfo.from_current_repository()
+        if git:
+            state.git = git
+        pr = PullRequestInfo(url=os.getenv("RECCE_PR_URL"))
+        state.pull_request = pr
+
+        return state
+
+    def sync_state(self, method: str):
+        """
+        Sync the state with the remote.
+
+        :param method: merge, revert, overwrite
+
+        """
+        if method == "merge":
+            self.state_loader.refresh()
+            self.import_state(self.state_loader.state, merge=True)
+            state = self.export_state()
+            self.state_loader.export(state)
+        elif method == "revert":
+            self.state_loader.refresh()
+            self.import_state(self.state_loader.state, merge=False)
+        elif method == "overwrite":
+            state = self.export_state()
+            self.state_loader.export(state)
+        else:
+            raise Exception(f"Unsupported method: {method}")
+
+    def _merge_checks(self, import_checks: list[Check]):
+        checks = list(self.checks)
+        imports = 0
+
+        def _calculate_checksum(c: Check):
+            payload = json.dumps(
+                {
+                    "type": str(c.type),
+                    "params": c.params,
+                    "view_options": c.view_options,
+                },
+                sort_keys=True,
+            )
+            return hashlib.sha256(payload.encode()).hexdigest()
+
+        checksum_map = {_calculate_checksum(c): c for c in self.checks if c.is_preset}
+        check_map = {c.check_id: c for c in self.checks}
+
+        # merge checks
+        for imported in import_checks:
+            check: Check = None
+            if imported.check_id in check_map:
+                check = check_map[imported.check_id]
+            elif imported.is_preset:
+                checksum = _calculate_checksum(imported)
+                if checksum in checksum_map:
+                    check = checksum_map[checksum]
+            if check:
+                is_merge = check.merge(imported)
+                if is_merge:
+                    imports += 1
+            else:
+                checks.append(imported)
+                imports += 1
+        self.checks = checks
+        return imports
+
+    def _merge_runs(self, import_runs: list[Run]):
+        runs = list(self.runs)
+        run_set = {run.run_id for run in self.runs}
+        imports = 0
+
+        for run in import_runs:
+            if run.run_id not in run_set:
+                runs.append(run)
+                imports += 1
+
+        runs.sort(key=lambda x: x.run_at)
+        self.runs = runs
+        return imports
+
+    def import_state(self, import_state: RecceState, merge: bool = True):
+        """
+        Import the state from another RecceState object.
+
+        :param import_state: the state to import
+        :param merge: whether to merge the state or replace the current state
+        """
+        import_runs = 0
+        import_checks = 0
+        if merge:
+            import_runs = self._merge_runs(import_state.runs)
+            import_checks = self._merge_checks(import_state.checks)
+        else:
+            self.runs = list(import_state.runs)
+            import_runs = len(self.runs)
+            self.checks = list(import_state.checks)
+            import_checks = len(self.checks)
+
+        # always merge for artifacts
+        if self.adapter:
+            self.adapter.import_artifacts(import_state.artifacts)
+
+        return import_runs, import_checks
+
+    def import_checks(self, import_state: RecceState, merge: bool = True):
+        """
+        Import the checks from another RecceState object.
+
+        :param import_state: the state to import
+        :param merge: whether to merge the state or replace the current state
+        """
+
+        if merge:
+            import_checks = self._merge_checks(import_state.checks)
+        else:
+            self.checks = list(import_state.checks)
+            import_checks = len(self.checks)
+
+        return import_checks
+
+    def mark_onboarding_completed(self):
+        if self.state_loader.cloud_mode:
+            try:
+                token = self.state_loader.cloud_options.get("github_token")
+                set_recce_cloud_onboarding_state(token, "completed")
+            except Exception as e:
+                logger.debug(f"Failed to mark onboarding completed in Recce Cloud. Reason: {str(e)}")
+        else:
+            # Skip the onboarding state for non-cloud mode
+            pass
+
+    @staticmethod
+    def verify_required_artifacts(**kwargs) -> Tuple[bool, Optional[str]]:
+        if kwargs.get("sqlmesh", False):
+            pass
+        else:
+            from recce.adapter.dbt_adapter import DbtAdapter
+
+            try:
+                DbtAdapter.load(**kwargs)
+            except FileNotFoundError as e:
+                return False, f"Cannot load the manifest: '{e.filename}'. Type 'recce debug'."
+
+        return True, None
+
+    def support_tasks(self):
+        return self.adapter.support_tasks()
+
+    def state_loader_mode(self):
+        """
+        The state loader mode is used for telemetry purpose.
+        """
+        if os.environ.get("DEMO", False):
+            return "demo"
+
+        if not self.state_loader:
+            return "none"
+
+        if self.state_loader.cloud_mode:
+            return "cloud"
+        elif self.state_loader.state_file:
+            return "file"
+        else:
+            return "none"
+
+
+recce_context: Optional[RecceContext] = None
+
+
+def load_context(**kwargs) -> RecceContext:
+    global recce_context
+    if recce_context is None:
+        recce_context = RecceContext.load(**kwargs)
+    return recce_context
+
+
+def default_context() -> RecceContext:
+    return recce_context
+
+
+def set_default_context(context: RecceContext):
+    """
+    Set the default context for the recce. This is for test purpose.
+    """
+
+    global recce_context
+    recce_context = context

package/recce-source/recce/diff.py

@@ -0,0 +1,26 @@
+import difflib
+
+import pandas as pd
+
+
+def diff_text(before: str, after: str):
+    if before is None and after is None:
+        print("not found in both states")
+        return
+    elif before == after:
+        print("no changes")
+        return
+
+    diff_output = difflib.unified_diff(before.splitlines(), after.splitlines(), "base", "current", lineterm="")
+    for line in diff_output:
+        print(line)
+
+
+def diff_dataframe(before: pd.DataFrame, after: pd.DataFrame):
+    if before is None and after is None:
+        print("not found in both states")
+        return
+
+    before_aligned, after_aligned = before.align(after)
+    diff = before_aligned.compare(after_aligned, result_names=("base", "current"))
+    print(diff.to_string(na_rep="-"))

package/recce-source/recce/event/CONFIG

@@ -0,0 +1 @@
+event_api_key: <API_KEY>

package/recce-source/recce/event/SENTRY_DNS

@@ -0,0 +1 @@
+https://64b0b12395a4f8596e081b26ca720719@o1081482.ingest.sentry.io/4506268402515968