@datarecce/ui 0.1.40 → 0.2.0

package/recce-source/recce/config.py

@@ -1,127 +0,0 @@
-# Configuration file for recce
-from rich.console import Console
-
-from recce import yaml
-from recce.exceptions import RecceConfigException
-from recce.util import SingletonMeta
-
-RECCE_CONFIG_FILE = "recce.yml"
-RECCE_PRESET_CHECK_COMMENT = """Preset Checks
-Please see https://docs.datarecce.io/features/preset-checks/
-"""
-RECCE_ERROR_LOG_FILE = "recce_error.log"
-console = Console()
-
-
-class RecceConfig(metaclass=SingletonMeta):
-    def __init__(self, config_file=RECCE_CONFIG_FILE):
-        self.config_file = config_file
-        self.config = None
-        self.load()
-
-    def load(self):
-        try:
-            with open(self.config_file, "r", encoding="utf-8") as f:
-                config = yaml.safe_load(f)
-                self.config = config if config else {}
-            self._verify_preset_checks()
-        except FileNotFoundError:
-            console.print(f"[[orange3]NOTICE[/orange3]] Generate default Recce config file at '{self.config_file}'")
-            self.config = self.generate_template()
-            self.save()
-
-    def _verify_preset_checks(self):
-        from recce.tasks.core import CheckValidator
-
-        if not self.config.get("checks"):
-            return
-
-        for check in self.config["checks"]:
-            try:
-                check_type = check.get("type")
-                if check_type is None:
-                    raise ValueError(f'Check type is required for check "{check}"')
-                if check_type == "lineage_diff":
-                    from recce.tasks.lineage import LineageDiffCheckValidator
-
-                    validator = LineageDiffCheckValidator()
-                elif check_type == "schema_diff":
-                    from recce.tasks.schema import SchemaDiffCheckValidator
-
-                    validator = SchemaDiffCheckValidator()
-                elif check_type == "row_count_diff":
-                    from recce.tasks.rowcount import RowCountDiffCheckValidator
-
-                    validator = RowCountDiffCheckValidator()
-                elif check_type == "query":
-                    from recce.tasks.query import QueryCheckValidator
-
-                    validator = QueryCheckValidator()
-                elif check_type == "query_diff":
-                    from recce.tasks.query import QueryDiffCheckValidator
-
-                    validator = QueryDiffCheckValidator()
-                elif check_type == "value_diff" or check_type == "value_diff_detail":
-                    from recce.tasks.valuediff import ValueDiffCheckValidator
-
-                    validator = ValueDiffCheckValidator()
-                elif check_type == "profile_diff":
-                    from recce.tasks.profile import ProfileCheckValidator
-
-                    validator = ProfileCheckValidator()
-                elif check_type == "top_k_diff":
-                    from recce.tasks.top_k import TopKDiffCheckValidator
-
-                    validator = TopKDiffCheckValidator()
-                elif check_type == "histogram_diff":
-                    from recce.tasks.histogram import HistogramDiffCheckValidator
-
-                    validator = HistogramDiffCheckValidator()
-                else:
-                    validator = CheckValidator()
-                validator.validate(check)
-            except Exception as e:
-                import json
-
-                raise RecceConfigException(
-                    f"Load preset checks failed from '{self.config_file}'\n{json.dumps(check, indent=2)}", cause=e
-                )
-
-    def generate_template(self):
-        data = yaml.CommentedMap(checks=yaml.CommentedSeq())
-        data.yaml_set_comment_before_after_key("checks", before=RECCE_PRESET_CHECK_COMMENT)
-        # Define default preset checks
-        default_checks = [
-            yaml.CommentedMap(
-                name="Row count diff",
-                description="Check the row count diff for all table models.",
-                type="row_count_diff",
-                params={"select": "state:modified,config.materialized:table"},
-            ),
-            yaml.CommentedMap(
-                name="Schema diff",
-                description="Check the schema diff for all nodes.",
-                type="schema_diff",
-            ),
-        ]
-
-        for check in default_checks:
-            data["checks"].append(check)
-
-        return data
-
-    def get(self, key, default=None):
-        return self.config.get(key, default)
-
-    def set(self, key, value):
-        self.config[key] = value
-
-    def save(self):
-        with open(RECCE_CONFIG_FILE, "w", encoding="utf-8") as f:
-            yaml.dump(self.config, f)
-
-    def __str__(self):
-        return yaml.dump(self.config)
-
-    def __repr__(self):
-        return self.__str__()

package/recce-source/recce/connect_to_cloud.py

@@ -1,138 +0,0 @@
-import base64
-import os.path
-import random
-import threading
-from http.server import BaseHTTPRequestHandler, HTTPServer
-from pathlib import Path
-from typing import Tuple
-from urllib.parse import parse_qs, urlparse
-
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import hashes, serialization
-from cryptography.hazmat.primitives.asymmetric import padding, rsa
-from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey
-from rich.console import Console
-
-from recce.event import update_recce_api_token
-from recce.exceptions import RecceConfigException
-from recce.util.onboarding_state import update_onboarding_state
-from recce.util.recce_cloud import RECCE_CLOUD_BASE_URL, RecceCloud
-
-console = Console()
-
-static_folder_path = Path(__file__).parent / "data"
-_server_lock = threading.Lock()
-_connection_url = None
-
-
-def decrypt_code(private_key: RSAPrivateKey, code: str) -> str:
-    ciphertext = base64.b64decode(code)
-    plaintext = private_key.decrypt(
-        ciphertext,
-        padding.OAEP(
-            mgf=padding.MGF1(algorithm=hashes.SHA1()),  # Node.js uses SHA1 by default
-            algorithm=hashes.SHA1(),
-            label=None,
-        ),
-    )
-    return plaintext.decode("utf-8")
-
-
-def handle_callback_request(query_string: str, private_key: RSAPrivateKey):
-    query_params = parse_qs(query_string)
-    code = query_params.get("code", [None])[0]
-    if not code:
-        raise RecceConfigException("Missing `code` in query")
-
-    api_token = decrypt_code(private_key, code)
-    if not RecceCloud(api_token).verify_token():
-        raise RecceConfigException("Invalid Recce Cloud API token")
-
-    update_recce_api_token(api_token)
-    update_onboarding_state(api_token, False)
-
-    return api_token  # for testability/debugging
-
-
-def make_callback_handler(private_key: RSAPrivateKey):
-    class OneTimeHTTPRequestHandler(BaseHTTPRequestHandler):
-        def do_GET(self):
-            try:
-                with open(os.path.join(static_folder_path, "auth_callback.html"), "r", encoding="utf-8") as f:
-                    callback_html_content = f.read()
-
-                # Parse query parameters
-                parsed_url = urlparse(self.path)
-
-                handle_callback_request(parsed_url.query, private_key)
-
-                # Construct HTML content
-                self.send_response(200)
-                self.send_header("Content-Type", "text/html")
-                self.send_header("Content-Length", str(len(callback_html_content.encode())))
-                self.end_headers()
-                self.wfile.write(callback_html_content.encode())
-
-            except Exception:
-                console.print_exception()
-                self.send_response(500)
-                self.end_headers()
-                self.wfile.write(b"<h1>Internal Server Error</h1>")
-            finally:
-                # Shut down the server after handling the first request
-                # Shutdown in a new thread to avoid deadlock
-                self.server.server_close()
-                threading.Thread(target=self.server.shutdown, daemon=True).start()
-
-        def log_message(self, format, *args):
-            # Suppress default logging
-            return
-
-    return OneTimeHTTPRequestHandler
-
-
-def is_callback_server_running():
-    return _server_lock.locked()
-
-
-def get_connection_url():
-    return _connection_url
-
-
-def run_one_time_http_server(private_key: RSAPrivateKey, port=8080):
-    handler = make_callback_handler(private_key)
-    server = HTTPServer(("localhost", port), handler)
-    server.serve_forever()
-
-
-def prepare_connection_url(public_key: RSAPublicKey):
-    public_key_pem_bytes = public_key.public_bytes(
-        encoding=serialization.Encoding.PEM,
-        format=serialization.PublicFormat.SubjectPublicKeyInfo,
-    )
-    public_key_pem_str = base64.b64encode(public_key_pem_bytes).decode("utf-8")
-    callback_port = random.randint(10000, 15000)
-    connect_url = f"{RECCE_CLOUD_BASE_URL}/connect?_key={public_key_pem_str}&_port={callback_port}"
-    return connect_url, callback_port
-
-
-def generate_key_pair() -> Tuple[RSAPrivateKey, RSAPublicKey]:
-    key_size = 2048  # Should be at least 2048
-
-    private_key = rsa.generate_private_key(
-        public_exponent=65537, key_size=key_size, backend=default_backend()  # Do not change
-    )
-
-    public_key = private_key.public_key()
-    return private_key, public_key
-
-
-def connect_to_cloud_background_task(private_key: RSAPrivateKey, callback_port, connection_url):
-    if is_callback_server_running():
-        return
-
-    with _server_lock:
-        global _connection_url
-        _connection_url = connection_url
-        run_one_time_http_server(private_key, callback_port)
-        _connection_url = None

package/recce-source/recce/core.py

@@ -1,334 +0,0 @@
-import hashlib
-import json
-import logging
-import os
-from dataclasses import dataclass, field
-from typing import Callable, Dict, List, Optional, Set, Tuple
-
-from recce.adapter.base import BaseAdapter
-from recce.models import Check, Run
-from recce.models.types import LineageDiff
-from recce.state import (
-    GitRepoInfo,
-    PullRequestInfo,
-    RecceState,
-    RecceStateLoader,
-    RecceStateMetadata,
-)
-from recce.util.recce_cloud import set_recce_cloud_onboarding_state
-
-logger = logging.getLogger("uvicorn")
-
-
-@dataclass
-class RecceContext:
-    review_mode: bool = False
-    adapter_type: str = None
-    adapter: BaseAdapter = None
-    state_loader: RecceStateLoader = None
-    runs: List[Run] = field(default_factory=list)
-    checks: List[Check] = field(default_factory=list)
-
-    @classmethod
-    def load(cls, **kwargs):
-        state_loader: RecceStateLoader = kwargs.get("state_loader")
-        is_review_mode = kwargs.get("review", False)
-
-        context = cls(
-            review_mode=is_review_mode,
-            state_loader=state_loader,
-        )
-
-        # Initiate the adapter
-        if kwargs.get("sqlmesh", False):
-            logger.warning("SQLMesh adapter is still in EXPERIMENTAL mode.")
-            from recce.adapter.sqlmesh_adapter import SqlmeshAdapter
-
-            context.adapter_type = "sqlmesh"
-            context.adapter = SqlmeshAdapter.load(**kwargs)
-        else:
-            from recce.adapter.dbt_adapter import DbtAdapter
-
-            context.adapter_type = "dbt"
-            context.adapter = DbtAdapter.load(**kwargs)
-
-        # Import state
-        if state_loader is not None:
-            state = state_loader.load()
-            if state:
-                context.import_state(state)
-
-            if is_review_mode:
-                if not state:
-                    raise Exception("The state file is required for review mode")
-
-        return context
-
-    def get_model(self, model_id: str, base=False):
-        return self.adapter.get_model(model_id, base=base)
-
-    def get_node_name_by_id(self, unique_id: object) -> object:
-        return self.adapter.get_node_name_by_id(unique_id)
-
-    def generate_sql(self, sql_template: str, base: bool = False, context: Dict = {}):
-        self.adapter.generate_sql(sql_template, base=base, context=context)
-
-    def get_lineage(self, base: Optional[bool] = False):
-        return self.adapter.get_lineage(base=base)
-
-    def get_lineage_diff(self) -> LineageDiff:
-        return self.adapter.get_lineage_diff()
-
-    def build_name_to_unique_id_index(self, excluded_types: Set = None) -> Dict[str, str]:
-        name_to_unique_id = {}
-        curr = self.get_lineage(base=False)
-        base = self.get_lineage(base=True)
-
-        for unique_id, node in curr["nodes"].items():
-            if excluded_types and node.get("resource_type") in excluded_types:
-                continue
-            name_to_unique_id[node["name"]] = unique_id
-        for unique_id, node in base["nodes"].items():
-            if excluded_types and node.get("resource_type") in excluded_types:
-                continue
-            name_to_unique_id[node["name"]] = unique_id
-        return name_to_unique_id
-
-    def start_monitor_artifacts(self, callback: Callable = None):
-        self.adapter.start_monitor_artifacts(callback=callback)
-
-    def stop_monitor_artifacts(self):
-        self.adapter.stop_monitor_artifacts()
-
-    def refresh_manifest(self, refresh_file_path: str = None):
-        self.adapter.refresh(refresh_file_path)
-
-    def start_monitor_base_env(self, callback: Callable = None):
-        self.adapter.start_monitor_base_env(callback=callback)
-
-    def stop_monitor_base_env(self):
-        self.adapter.stop_monitor_base_env()
-
-    def export_state(self) -> RecceState:
-        """
-        Export the state to a RecceState object.
-        """
-        state = RecceState()
-        state.metadata = RecceStateMetadata()
-
-        # runs & checks & artifacts
-        state.runs = self.runs
-        state.checks = self.checks
-        state.artifacts = self.adapter.export_artifacts()
-
-        # git & pull_request. If in review mode, use the review state
-        if self.review_mode:
-            state.git = self.state_loader.state.git
-            state.pull_request = self.state_loader.state.pull_request
-        else:
-            git = GitRepoInfo.from_current_repository()
-            if git:
-                state.git = git
-            if self.state_loader.pr_info:
-                state.pull_request = self.state_loader.pr_info
-
-        return state
-
-    def export_demo_state(self) -> RecceState:
-        """
-        Export the demo state to a RecceState object for the demo sites.
-        """
-        state = RecceState()
-        state.metadata = RecceStateMetadata()
-
-        # runs & checks
-        state.runs = self.runs
-        state.checks = self.checks
-        state.artifacts = self.adapter.export_artifacts()
-        git = GitRepoInfo.from_current_repository()
-        if git:
-            state.git = git
-        pr = PullRequestInfo(url=os.getenv("RECCE_PR_URL"))
-        state.pull_request = pr
-
-        return state
-
-    def sync_state(self, method: str):
-        """
-        Sync the state with the remote.
-
-        :param method: merge, revert, overwrite
-
-        """
-        if method == "merge":
-            self.state_loader.refresh()
-            self.import_state(self.state_loader.state, merge=True)
-            state = self.export_state()
-            self.state_loader.export(state)
-        elif method == "revert":
-            self.state_loader.refresh()
-            self.import_state(self.state_loader.state, merge=False)
-        elif method == "overwrite":
-            state = self.export_state()
-            self.state_loader.export(state)
-        else:
-            raise Exception(f"Unsupported method: {method}")
-
-    def _merge_checks(self, import_checks: list[Check]):
-        checks = list(self.checks)
-        imports = 0
-
-        def _calculate_checksum(c: Check):
-            payload = json.dumps(
-                {
-                    "type": str(c.type),
-                    "params": c.params,
-                    "view_options": c.view_options,
-                },
-                sort_keys=True,
-            )
-            return hashlib.sha256(payload.encode()).hexdigest()
-
-        checksum_map = {_calculate_checksum(c): c for c in self.checks if c.is_preset}
-        check_map = {c.check_id: c for c in self.checks}
-
-        # merge checks
-        for imported in import_checks:
-            check: Check = None
-            if imported.check_id in check_map:
-                check = check_map[imported.check_id]
-            elif imported.is_preset:
-                checksum = _calculate_checksum(imported)
-                if checksum in checksum_map:
-                    check = checksum_map[checksum]
-            if check:
-                is_merge = check.merge(imported)
-                if is_merge:
-                    imports += 1
-            else:
-                checks.append(imported)
-                imports += 1
-        self.checks = checks
-        return imports
-
-    def _merge_runs(self, import_runs: list[Run]):
-        runs = list(self.runs)
-        run_set = {run.run_id for run in self.runs}
-        imports = 0
-
-        for run in import_runs:
-            if run.run_id not in run_set:
-                runs.append(run)
-                imports += 1
-
-        runs.sort(key=lambda x: x.run_at)
-        self.runs = runs
-        return imports
-
-    def import_state(self, import_state: RecceState, merge: bool = True):
-        """
-        Import the state from another RecceState object.
-
-        :param import_state: the state to import
-        :param merge: whether to merge the state or replace the current state
-        """
-        import_runs = 0
-        import_checks = 0
-        if merge:
-            import_runs = self._merge_runs(import_state.runs)
-            import_checks = self._merge_checks(import_state.checks)
-        else:
-            self.runs = list(import_state.runs)
-            import_runs = len(self.runs)
-            self.checks = list(import_state.checks)
-            import_checks = len(self.checks)
-
-        # always merge for artifacts
-        if self.adapter:
-            self.adapter.import_artifacts(import_state.artifacts)
-
-        return import_runs, import_checks
-
-    def import_checks(self, import_state: RecceState, merge: bool = True):
-        """
-        Import the checks from another RecceState object.
-
-        :param import_state: the state to import
-        :param merge: whether to merge the state or replace the current state
-        """
-
-        if merge:
-            import_checks = self._merge_checks(import_state.checks)
-        else:
-            self.checks = list(import_state.checks)
-            import_checks = len(self.checks)
-
-        return import_checks
-
-    def mark_onboarding_completed(self):
-        if self.state_loader.cloud_mode:
-            try:
-                token = self.state_loader.cloud_options.get("github_token")
-                set_recce_cloud_onboarding_state(token, "completed")
-            except Exception as e:
-                logger.debug(f"Failed to mark onboarding completed in Recce Cloud. Reason: {str(e)}")
-        else:
-            # Skip the onboarding state for non-cloud mode
-            pass
-
-    @staticmethod
-    def verify_required_artifacts(**kwargs) -> Tuple[bool, Optional[str]]:
-        if kwargs.get("sqlmesh", False):
-            pass
-        else:
-            from recce.adapter.dbt_adapter import DbtAdapter
-
-            try:
-                DbtAdapter.load(**kwargs)
-            except FileNotFoundError as e:
-                return False, f"Cannot load the manifest: '{e.filename}'. Type 'recce debug'."
-
-        return True, None
-
-    def support_tasks(self):
-        return self.adapter.support_tasks()
-
-    def state_loader_mode(self):
-        """
-        The state loader mode is used for telemetry purpose.
-        """
-        if os.environ.get("DEMO", False):
-            return "demo"
-
-        if not self.state_loader:
-            return "none"
-
-        if self.state_loader.cloud_mode:
-            return "cloud"
-        elif self.state_loader.state_file:
-            return "file"
-        else:
-            return "none"
-
-
-recce_context: Optional[RecceContext] = None
-
-
-def load_context(**kwargs) -> RecceContext:
-    global recce_context
-    if recce_context is None:
-        recce_context = RecceContext.load(**kwargs)
-    return recce_context
-
-
-def default_context() -> RecceContext:
-    return recce_context
-
-
-def set_default_context(context: RecceContext):
-    """
-    Set the default context for the recce. This is for test purpose.
-    """
-
-    global recce_context
-    recce_context = context

package/recce-source/recce/diff.py

@@ -1,26 +0,0 @@
-import difflib
-
-import pandas as pd
-
-
-def diff_text(before: str, after: str):
-    if before is None and after is None:
-        print("not found in both states")
-        return
-    elif before == after:
-        print("no changes")
-        return
-
-    diff_output = difflib.unified_diff(before.splitlines(), after.splitlines(), "base", "current", lineterm="")
-    for line in diff_output:
-        print(line)
-
-
-def diff_dataframe(before: pd.DataFrame, after: pd.DataFrame):
-    if before is None and after is None:
-        print("not found in both states")
-        return
-
-    before_aligned, after_aligned = before.align(after)
-    diff = before_aligned.compare(after_aligned, result_names=("base", "current"))
-    print(diff.to_string(na_rep="-"))

package/recce-source/recce/event/CONFIG

@@ -1 +0,0 @@
-event_api_key: <API_KEY>

package/recce-source/recce/event/SENTRY_DNS

@@ -1 +0,0 @@
-https://64b0b12395a4f8596e081b26ca720719@o1081482.ingest.sentry.io/4506268402515968