@datarecce/ui 0.1.40 → 0.2.0

package/recce-source/recce_cloud/pyproject.toml

@@ -1,49 +0,0 @@
-[project]
-name = "recce-cloud"
-dynamic = ["version", "readme"]
-description = "Lightweight CLI for Recce Cloud operations"
-authors = [{name = "InfuseAI Dev Team", email = "dev@infuseai.io"}]
-requires-python = ">=3.9"
-license = {text = "Apache-2.0"}
-
-dependencies = [
-    "click>=7.1",
-    "requests>=2.28.1",
-    "rich>=12.0.0",
-]
-
-classifiers = [
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
-    "Programming Language :: Python :: 3.13",
-    "License :: OSI Approved :: Apache Software License",
-    "Operating System :: OS Independent",
-    "Development Status :: 4 - Beta",
-]
-
-[project.scripts]
-recce-cloud = "recce_cloud.cli:cloud_cli"
-
-[project.urls]
-"Bug Tracker" = "https://github.com/InfuseAI/recce/issues"
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[tool.hatch.version]
-path = "VERSION"
-pattern = "(?P<version>.+)"
-
-[tool.hatch.metadata]
-allow-direct-references = true
-
-[tool.hatch.metadata.hooks.custom]
-
-[tool.hatch.build.targets.wheel]
-packages = ["."]
-
-[tool.hatch.build.targets.wheel.sources]
-"." = "recce_cloud"

package/recce-source/recce_cloud/upload.py

@@ -1,214 +0,0 @@
-"""
-Upload helper functions for recce-cloud CLI.
-"""
-
-import os
-import sys
-
-import requests
-
-from recce_cloud.api.client import RecceCloudClient
-from recce_cloud.api.exceptions import RecceCloudException
-from recce_cloud.api.factory import create_platform_client
-
-
-def upload_to_existing_session(
-    console, token: str, session_id: str, manifest_path: str, catalog_path: str, adapter_type: str, target_path: str
-):
-    """
-    Upload artifacts to an existing Recce Cloud session using session ID.
-
-    This is the generic workflow that requires a pre-existing session ID.
-    """
-    try:
-        client = RecceCloudClient(token)
-    except Exception as e:
-        console.print("[red]Error:[/red] Failed to initialize API client")
-        console.print(f"Reason: {e}")
-        sys.exit(2)
-
-    # Get session info (org_id, project_id)
-    console.print(f'Uploading artifacts for session ID "{session_id}"')
-    try:
-        session = client.get_session(session_id)
-        if session.get("status") == "error":
-            console.print(f"[red]Error:[/red] {session.get('message')}")
-            sys.exit(2)
-
-        org_id = session.get("org_id")
-        if org_id is None:
-            console.print(f"[red]Error:[/red] Session ID {session_id} does not belong to any organization.")
-            sys.exit(2)
-
-        project_id = session.get("project_id")
-        if project_id is None:
-            console.print(f"[red]Error:[/red] Session ID {session_id} does not belong to any project.")
-            sys.exit(2)
-
-    except RecceCloudException as e:
-        console.print("[red]Error:[/red] Failed to get session info")
-        console.print(f"Reason: {e.reason}")
-        sys.exit(2)
-    except Exception as e:
-        console.print("[red]Error:[/red] Failed to get session info")
-        console.print(f"Reason: {e}")
-        sys.exit(2)
-
-    # Get presigned URLs
-    try:
-        presigned_urls = client.get_upload_urls_by_session_id(org_id, project_id, session_id)
-    except RecceCloudException as e:
-        console.print("[red]Error:[/red] Failed to get upload URLs")
-        console.print(f"Reason: {e.reason}")
-        sys.exit(4)
-    except Exception as e:
-        console.print("[red]Error:[/red] Failed to get upload URLs")
-        console.print(f"Reason: {e}")
-        sys.exit(4)
-
-    # Upload manifest.json
-    console.print(f'Uploading manifest from path "{manifest_path}"')
-    try:
-        with open(manifest_path, "rb") as f:
-            response = requests.put(presigned_urls["manifest_url"], data=f.read())
-        if response.status_code not in [200, 204]:
-            raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
-    except Exception as e:
-        console.print("[red]Error:[/red] Failed to upload manifest.json")
-        console.print(f"Reason: {e}")
-        sys.exit(4)
-
-    # Upload catalog.json
-    console.print(f'Uploading catalog from path "{catalog_path}"')
-    try:
-        with open(catalog_path, "rb") as f:
-            response = requests.put(presigned_urls["catalog_url"], data=f.read())
-        if response.status_code not in [200, 204]:
-            raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
-    except Exception as e:
-        console.print("[red]Error:[/red] Failed to upload catalog.json")
-        console.print(f"Reason: {e}")
-        sys.exit(4)
-
-    # Update session metadata
-    try:
-        client.update_session(org_id, project_id, session_id, adapter_type)
-    except RecceCloudException as e:
-        console.print("[red]Error:[/red] Failed to update session metadata")
-        console.print(f"Reason: {e.reason}")
-        sys.exit(4)
-    except Exception as e:
-        console.print("[red]Error:[/red] Failed to update session metadata")
-        console.print(f"Reason: {e}")
-        sys.exit(4)
-
-    # Success!
-    console.rule("Uploaded Successfully", style="green")
-    console.print(
-        f'Uploaded dbt artifacts to Recce Cloud for session ID "{session_id}" from "{os.path.abspath(target_path)}"'
-    )
-    sys.exit(0)
-
-
-def upload_with_platform_apis(
-    console, token: str, ci_info, manifest_path: str, catalog_path: str, adapter_type: str, target_path: str
-):
-    """
-    Upload artifacts using platform-specific APIs (GitHub Actions or GitLab CI).
-
-    This workflow uses touch-recce-session to create a session automatically.
-    """
-    # Validate platform support
-    if ci_info.platform not in ["github-actions", "gitlab-ci"]:
-        console.print("[red]Error:[/red] Platform-specific upload requires GitHub Actions or GitLab CI environment")
-        console.print(f"Detected platform: {ci_info.platform or 'unknown'}")
-        console.print(
-            "Either run this command in a supported CI environment or provide --session-id for generic upload"
-        )
-        sys.exit(1)
-
-    # Create platform-specific client
-    try:
-        client = create_platform_client(token, ci_info)
-    except ValueError as e:
-        console.print("[red]Error:[/red] Failed to create platform client")
-        console.print(f"Reason: {e}")
-        sys.exit(2)
-
-    # Touch session to create or get session ID
-    console.rule("Creating/touching session", style="blue")
-    try:
-        session_response = client.touch_recce_session(
-            branch=ci_info.source_branch or ci_info.base_branch or "main",
-            adapter_type=adapter_type,
-            cr_number=ci_info.cr_number,
-            commit_sha=ci_info.commit_sha,
-            session_type=ci_info.session_type,
-        )
-
-        session_id = session_response.get("session_id")
-        manifest_upload_url = session_response.get("manifest_upload_url")
-        catalog_upload_url = session_response.get("catalog_upload_url")
-
-        if not session_id or not manifest_upload_url or not catalog_upload_url:
-            console.print("[red]Error:[/red] Incomplete response from touch-recce-session API")
-            console.print(f"Response: {session_response}")
-            sys.exit(4)
-
-        console.print(f"[green]Session ID:[/green] {session_id}")
-
-    except RecceCloudException as e:
-        console.print("[red]Error:[/red] Failed to create/touch session")
-        console.print(f"Reason: {e.reason}")
-        sys.exit(4)
-    except Exception as e:
-        console.print("[red]Error:[/red] Failed to create/touch session")
-        console.print(f"Reason: {e}")
-        sys.exit(4)
-
-    # Upload manifest.json
-    console.print(f'Uploading manifest from path "{manifest_path}"')
-    try:
-        with open(manifest_path, "rb") as f:
-            response = requests.put(manifest_upload_url, data=f.read())
-        if response.status_code not in [200, 204]:
-            raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
-    except Exception as e:
-        console.print("[red]Error:[/red] Failed to upload manifest.json")
-        console.print(f"Reason: {e}")
-        sys.exit(4)
-
-    # Upload catalog.json
-    console.print(f'Uploading catalog from path "{catalog_path}"')
-    try:
-        with open(catalog_path, "rb") as f:
-            response = requests.put(catalog_upload_url, data=f.read())
-        if response.status_code not in [200, 204]:
-            raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
-    except Exception as e:
-        console.print("[red]Error:[/red] Failed to upload catalog.json")
-        console.print(f"Reason: {e}")
-        sys.exit(4)
-
-    # Notify upload completion
-    console.print("Notifying upload completion...")
-    try:
-        client.upload_completed(session_id=session_id, commit_sha=ci_info.commit_sha)
-    except RecceCloudException as e:
-        console.print("[yellow]Warning:[/yellow] Failed to notify upload completion")
-        console.print(f"Reason: {e.reason}")
-        # Non-fatal, continue
-    except Exception as e:
-        console.print("[yellow]Warning:[/yellow] Failed to notify upload completion")
-        console.print(f"Reason: {e}")
-        # Non-fatal, continue
-
-    # Success!
-    console.rule("Uploaded Successfully", style="green")
-    console.print(f'Uploaded dbt artifacts to Recce Cloud for session ID "{session_id}"')
-    console.print(f'Artifacts from: "{os.path.abspath(target_path)}"')
-
-    if ci_info.cr_url:
-        console.print(f"Change request: {ci_info.cr_url}")
-
-    sys.exit(0)

package/recce-source/tests/adapter/dbt_adapter/conftest.py

@@ -1,17 +0,0 @@
-from unittest.mock import patch
-
-import pytest
-
-from recce.core import set_default_context
-
-from .dbt_test_helper import DbtTestHelper
-
-
-@pytest.fixture
-def dbt_test_helper():
-    with patch("recce.adapter.dbt_adapter.log_performance"):
-        helper = DbtTestHelper()
-        context = helper.context
-        set_default_context(context)
-        yield helper
-        helper.cleanup()

package/recce-source/tests/adapter/dbt_adapter/dbt_test_helper.py

@@ -1,298 +0,0 @@
-import os
-import textwrap
-import uuid
-from datetime import datetime
-from io import StringIO
-
-from dbt.contracts.graph.nodes import (
-    ModelNode,
-    SeedNode,
-    SnapshotNode,
-    SourceDefinition,
-)
-from dbt.contracts.results import (
-    CatalogArtifact,
-    CatalogTable,
-    ColumnMetadata,
-    TableMetadata,
-)
-
-from recce.adapter.dbt_adapter import DbtAdapter, as_manifest, load_manifest
-from recce.core import RecceContext
-
-
-class DbtTestHelper:
-
-    def __init__(self):
-        schema_prefix = "schema_" + uuid.uuid4().hex
-        self.base_schema = f"{schema_prefix}_base"
-        self.curr_schema = f"{schema_prefix}_curr"
-
-        current_dir = os.path.dirname(os.path.abspath(__file__))
-        project_dir = os.path.join(current_dir, "test_proj")
-        profiles_dir = project_dir
-        manifest_path = os.path.join(project_dir, "manifest.json")
-
-        dbt_adapter = DbtAdapter.load(
-            no_artifacts=True,
-            project_dir=project_dir,
-            profiles_dir=profiles_dir,
-        )
-
-        context = RecceContext()
-        context.adapter_type = "dbt"
-        context.adapter = dbt_adapter
-        context.schema_prefix = schema_prefix
-        self.adapter = dbt_adapter
-        self.context = context
-        curr_writable_manifest = load_manifest(manifest_path)
-        base_writable_manifest = load_manifest(manifest_path)
-
-        self.curr_manifest = as_manifest(curr_writable_manifest)
-        self.base_manifest = as_manifest(base_writable_manifest)
-        now = datetime.now()
-        self.curr_catalog = CatalogArtifact.from_results(
-            generated_at=now,
-            nodes={},
-            sources={},
-            compile_results=None,
-            errors=None,
-        )
-        self.base_catalog = CatalogArtifact.from_results(
-            generated_at=now,
-            nodes={},
-            sources={},
-            compile_results=None,
-            errors=None,
-        )
-        self.context = context
-
-        self.adapter.execute(f"CREATE schema IF NOT EXISTS {self.base_schema}")
-        self.adapter.execute(f"CREATE schema IF NOT EXISTS {self.curr_schema}")
-        self.adapter.set_artifacts(
-            base_writable_manifest,
-            curr_writable_manifest,
-            self.curr_manifest,
-            self.base_manifest,
-            self.base_catalog,
-            self.curr_catalog,
-        )
-
-    def create_model(
-        self,
-        model_name,
-        base_csv=None,
-        curr_csv=None,
-        base_sql=None,
-        curr_sql=None,
-        depends_on=None,
-        disabled=False,
-        unique_id=None,
-        resource_type="model",
-        package_name="recce_test",
-        base_columns: dict[str, str] = None,
-        curr_columns: dict[str, str] = None,
-        patch_func=None,
-    ):
-        # unique_id = f"model.{package_name}.{model_name}"
-        unique_id = unique_id if unique_id else model_name
-        if depends_on is None:
-            depends_on = []
-
-        def _add_model_to_manifest(base):
-            if base:
-                schema = self.base_schema
-                manifest = self.base_manifest
-                catalog = self.base_catalog
-                csv = base_csv
-                sql = base_sql
-                columns = base_columns
-            else:
-                schema = self.curr_schema
-                manifest = self.curr_manifest
-                catalog = self.curr_catalog
-                csv = curr_csv
-                sql = curr_sql
-                columns = curr_columns
-
-            if csv:
-                dbt_adapter = self.adapter
-                csv = textwrap.dedent(csv)
-                with dbt_adapter.connection_named("create model"):
-                    import pandas as pd
-
-                    df = pd.read_csv(StringIO(csv))  # noqa: F841
-                    # DuckDB disables python_scan_all_frames by default after 1.1,
-                    # ref: https://github.com/duckdb/duckdb/pull/13896
-                    dbt_adapter.execute("SET python_scan_all_frames=true")
-                    dbt_adapter.execute(f"CREATE TABLE {schema}.{model_name} AS SELECT * FROM df")
-            raw_code = sql if sql else csv
-
-            if columns:
-                index = 1
-                table = CatalogTable(TableMetadata(type="BASE TABLE", schema=schema, name=model_name), {}, {})
-                catalog.nodes[unique_id] = table
-                for column, column_type in columns.items():
-                    col_data = ColumnMetadata(type=column_type, index=index, name=column)
-                    catalog.nodes[unique_id].columns[column] = col_data
-                    index = index + 1
-
-            node_dict = {
-                "resource_type": resource_type,
-                "name": model_name,
-                "package_name": package_name,
-                "path": "",
-                "original_file_path": "",
-                "unique_id": unique_id,
-                "fqn": [
-                    package_name,
-                    model_name,
-                ],
-                "schema": schema,
-                "alias": model_name,
-                "checksum": {
-                    "name": "sha256",
-                    "checksum": hash(raw_code),
-                },
-                "raw_code": raw_code,
-                "config": {
-                    "materialized": "table",
-                    "tags": ["test_tag"],
-                },
-                "tags": ["test_tag"],
-                "depends_on": {"nodes": depends_on},
-            }
-            if patch_func:
-                patch_func(node_dict)
-
-            if resource_type == "snapshot":
-                node = SnapshotNode.from_dict(node_dict)
-            elif resource_type == "seed":
-                node = SeedNode.from_dict(node_dict)
-            else:
-                node = ModelNode.from_dict(node_dict)
-
-            if disabled:
-                manifest.add_disabled_nofile(node)
-            else:
-                manifest.add_node_nofile(node)
-            return node
-
-        if base_csv or base_sql:
-            _add_model_to_manifest(True)
-
-        if curr_csv or curr_sql:
-            _add_model_to_manifest(False)
-
-        self.adapter.set_artifacts(
-            self.base_manifest.writable_manifest(),
-            self.curr_manifest.writable_manifest(),
-            self.curr_manifest,
-            self.base_manifest,
-            self.base_catalog,
-            self.curr_catalog,
-        )
-
-    def remove_model(self, model_name):
-        dbt_adapter = self.adapter
-        with dbt_adapter.connection_named("cleanup"):
-            dbt_adapter.execute(f"DROP TABLE IF EXISTS {self.base_schema}.{model_name}")
-            dbt_adapter.execute(f"DROP TABLE IF EXISTS {self.curr_schema}.{model_name} ")
-
-    def create_source(
-        self,
-        source_name,
-        table_name,
-        base_csv=None,
-        curr_csv=None,
-        unique_id=None,
-        package_name="recce_test",
-        base_columns: dict[str, str] = None,
-        curr_columns: dict[str, str] = None,
-        patch_func=None,
-    ):
-        unique_id = unique_id if unique_id else f"source.{package_name}.{source_name}.{table_name}"
-
-        def _add_source_to_manifest(base):
-            if base:
-                schema = self.base_schema
-                manifest = self.base_manifest
-                catalog = self.base_catalog
-                csv = base_csv
-                columns = base_columns
-            else:
-                schema = self.curr_schema
-                manifest = self.curr_manifest
-                catalog = self.curr_catalog
-                csv = curr_csv
-                columns = curr_columns
-
-            if csv:
-                dbt_adapter = self.adapter
-                csv = textwrap.dedent(csv)
-                with dbt_adapter.connection_named("create source"):
-                    import pandas as pd
-
-                    df = pd.read_csv(StringIO(csv))  # noqa: F841
-                    # DuckDB disables python_scan_all_frames by default after 1.1,
-                    # ref: https://github.com/duckdb/duckdb/pull/13896
-                    dbt_adapter.execute("SET python_scan_all_frames=true")
-                    dbt_adapter.execute(f"CREATE TABLE {schema}.{table_name} AS SELECT * FROM df")
-
-            if columns:
-                index = 1
-                table = CatalogTable(TableMetadata(type="BASE TABLE", schema=schema, name=table_name), {}, {})
-                catalog.sources[unique_id] = table
-                for column, column_type in columns.items():
-                    col_data = ColumnMetadata(type=column_type, index=index, name=column)
-                    catalog.sources[unique_id].columns[column] = col_data
-                    index = index + 1
-
-            node_dict = {
-                "resource_type": "source",
-                "source_name": source_name,
-                "name": table_name,
-                "identifier": table_name,
-                "source_description": "test source",
-                "loader": "",
-                "package_name": package_name,
-                "path": "",
-                "original_file_path": "",
-                "unique_id": unique_id,
-                "fqn": [
-                    package_name,
-                    source_name,
-                    table_name,
-                ],
-                "schema": schema,
-            }
-            if patch_func:
-                patch_func(node_dict)
-
-            source = SourceDefinition.from_dict(node_dict)
-            manifest.sources[unique_id] = source
-            return source
-
-        if base_csv:
-            _add_source_to_manifest(True)
-
-        if curr_csv:
-            _add_source_to_manifest(False)
-
-        self.adapter.set_artifacts(
-            self.base_manifest.writable_manifest(),
-            self.curr_manifest.writable_manifest(),
-            self.curr_manifest,
-            self.base_manifest,
-            self.base_catalog,
-            self.curr_catalog,
-        )
-
-    def cleanup(self):
-        dbt_adapter = self.adapter
-        with dbt_adapter.connection_named("cleanup"):
-            dbt_adapter.execute(f"DROP SCHEMA IF EXISTS {self.base_schema} CASCADE")
-            dbt_adapter.execute(f"DROP SCHEMA IF EXISTS {self.curr_schema} CASCADE")
-
-    def create_snapshot(self, sanpshot_name, base_csv, curr_csv, depends_on=[]):
-        self.create_model(sanpshot_name, base_csv, curr_csv, depends_on=depends_on, resource_type="snapshot")

package/recce-source/tests/adapter/dbt_adapter/test_dbt_adapter.py

@@ -1,25 +0,0 @@
-from recce.adapter.dbt_adapter import DbtAdapter, dbt_supported_registry
-
-
-def test_dbt_adapter_support_tasks(dbt_test_helper):
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    # Test dbt task support
-    support_tasks = adapter.support_tasks()
-
-    for task_type in dbt_supported_registry:
-        task = task_type.value
-        assert task in support_tasks
-        assert support_tasks[task] is True
-
-
-def test_dbt_adapter_support_tasks_without_required_dbt_package(dbt_test_helper):
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    # Mock the macros in manifest to simulate no required dbt package installed
-    adapter.manifest.macros = {}
-
-    support_tasks = adapter.support_tasks()
-
-    for task_type in dbt_supported_registry:
-        task = task_type.value
-        assert task in support_tasks