@datarecce/ui 0.1.30 → 0.1.31

package/recce-source/recce_cloud/pyproject.toml

@@ -0,0 +1,49 @@
+[project]
+name = "recce-cloud"
+dynamic = ["version", "readme"]
+description = "Lightweight CLI for Recce Cloud operations"
+authors = [{name = "InfuseAI Dev Team", email = "dev@infuseai.io"}]
+requires-python = ">=3.9"
+license = {text = "Apache-2.0"}
+
+dependencies = [
+    "click>=7.1",
+    "requests>=2.28.1",
+    "rich>=12.0.0",
+]
+
+classifiers = [
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Development Status :: 4 - Beta",
+]
+
+[project.scripts]
+recce-cloud = "recce_cloud.cli:cloud_cli"
+
+[project.urls]
+"Bug Tracker" = "https://github.com/InfuseAI/recce/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "VERSION"
+pattern = "(?P<version>.+)"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.metadata.hooks.custom]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.hatch.build.targets.wheel.sources]
+"." = "recce_cloud"

package/recce-source/recce_cloud/upload.py

@@ -0,0 +1,214 @@
+"""
+Upload helper functions for recce-cloud CLI.
+"""
+
+import os
+import sys
+
+import requests
+
+from recce_cloud.api.client import RecceCloudClient
+from recce_cloud.api.exceptions import RecceCloudException
+from recce_cloud.api.factory import create_platform_client
+
+
+def upload_to_existing_session(
+    console, token: str, session_id: str, manifest_path: str, catalog_path: str, adapter_type: str, target_path: str
+):
+    """
+    Upload artifacts to an existing Recce Cloud session using session ID.
+
+    This is the generic workflow that requires a pre-existing session ID.
+    """
+    try:
+        client = RecceCloudClient(token)
+    except Exception as e:
+        console.print("[red]Error:[/red] Failed to initialize API client")
+        console.print(f"Reason: {e}")
+        sys.exit(2)
+
+    # Get session info (org_id, project_id)
+    console.print(f'Uploading artifacts for session ID "{session_id}"')
+    try:
+        session = client.get_session(session_id)
+        if session.get("status") == "error":
+            console.print(f"[red]Error:[/red] {session.get('message')}")
+            sys.exit(2)
+
+        org_id = session.get("org_id")
+        if org_id is None:
+            console.print(f"[red]Error:[/red] Session ID {session_id} does not belong to any organization.")
+            sys.exit(2)
+
+        project_id = session.get("project_id")
+        if project_id is None:
+            console.print(f"[red]Error:[/red] Session ID {session_id} does not belong to any project.")
+            sys.exit(2)
+
+    except RecceCloudException as e:
+        console.print("[red]Error:[/red] Failed to get session info")
+        console.print(f"Reason: {e.reason}")
+        sys.exit(2)
+    except Exception as e:
+        console.print("[red]Error:[/red] Failed to get session info")
+        console.print(f"Reason: {e}")
+        sys.exit(2)
+
+    # Get presigned URLs
+    try:
+        presigned_urls = client.get_upload_urls_by_session_id(org_id, project_id, session_id)
+    except RecceCloudException as e:
+        console.print("[red]Error:[/red] Failed to get upload URLs")
+        console.print(f"Reason: {e.reason}")
+        sys.exit(4)
+    except Exception as e:
+        console.print("[red]Error:[/red] Failed to get upload URLs")
+        console.print(f"Reason: {e}")
+        sys.exit(4)
+
+    # Upload manifest.json
+    console.print(f'Uploading manifest from path "{manifest_path}"')
+    try:
+        with open(manifest_path, "rb") as f:
+            response = requests.put(presigned_urls["manifest_url"], data=f.read())
+        if response.status_code not in [200, 204]:
+            raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
+    except Exception as e:
+        console.print("[red]Error:[/red] Failed to upload manifest.json")
+        console.print(f"Reason: {e}")
+        sys.exit(4)
+
+    # Upload catalog.json
+    console.print(f'Uploading catalog from path "{catalog_path}"')
+    try:
+        with open(catalog_path, "rb") as f:
+            response = requests.put(presigned_urls["catalog_url"], data=f.read())
+        if response.status_code not in [200, 204]:
+            raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
+    except Exception as e:
+        console.print("[red]Error:[/red] Failed to upload catalog.json")
+        console.print(f"Reason: {e}")
+        sys.exit(4)
+
+    # Update session metadata
+    try:
+        client.update_session(org_id, project_id, session_id, adapter_type)
+    except RecceCloudException as e:
+        console.print("[red]Error:[/red] Failed to update session metadata")
+        console.print(f"Reason: {e.reason}")
+        sys.exit(4)
+    except Exception as e:
+        console.print("[red]Error:[/red] Failed to update session metadata")
+        console.print(f"Reason: {e}")
+        sys.exit(4)
+
+    # Success!
+    console.rule("Uploaded Successfully", style="green")
+    console.print(
+        f'Uploaded dbt artifacts to Recce Cloud for session ID "{session_id}" from "{os.path.abspath(target_path)}"'
+    )
+    sys.exit(0)
+
+
+def upload_with_platform_apis(
+    console, token: str, ci_info, manifest_path: str, catalog_path: str, adapter_type: str, target_path: str
+):
+    """
+    Upload artifacts using platform-specific APIs (GitHub Actions or GitLab CI).
+
+    This workflow uses touch-recce-session to create a session automatically.
+    """
+    # Validate platform support
+    if ci_info.platform not in ["github-actions", "gitlab-ci"]:
+        console.print("[red]Error:[/red] Platform-specific upload requires GitHub Actions or GitLab CI environment")
+        console.print(f"Detected platform: {ci_info.platform or 'unknown'}")
+        console.print(
+            "Either run this command in a supported CI environment or provide --session-id for generic upload"
+        )
+        sys.exit(1)
+
+    # Create platform-specific client
+    try:
+        client = create_platform_client(token, ci_info)
+    except ValueError as e:
+        console.print("[red]Error:[/red] Failed to create platform client")
+        console.print(f"Reason: {e}")
+        sys.exit(2)
+
+    # Touch session to create or get session ID
+    console.rule("Creating/touching session", style="blue")
+    try:
+        session_response = client.touch_recce_session(
+            branch=ci_info.source_branch or ci_info.base_branch or "main",
+            adapter_type=adapter_type,
+            cr_number=ci_info.cr_number,
+            commit_sha=ci_info.commit_sha,
+            session_type=ci_info.session_type,
+        )
+
+        session_id = session_response.get("session_id")
+        manifest_upload_url = session_response.get("manifest_upload_url")
+        catalog_upload_url = session_response.get("catalog_upload_url")
+
+        if not session_id or not manifest_upload_url or not catalog_upload_url:
+            console.print("[red]Error:[/red] Incomplete response from touch-recce-session API")
+            console.print(f"Response: {session_response}")
+            sys.exit(4)
+
+        console.print(f"[green]Session ID:[/green] {session_id}")
+
+    except RecceCloudException as e:
+        console.print("[red]Error:[/red] Failed to create/touch session")
+        console.print(f"Reason: {e.reason}")
+        sys.exit(4)
+    except Exception as e:
+        console.print("[red]Error:[/red] Failed to create/touch session")
+        console.print(f"Reason: {e}")
+        sys.exit(4)
+
+    # Upload manifest.json
+    console.print(f'Uploading manifest from path "{manifest_path}"')
+    try:
+        with open(manifest_path, "rb") as f:
+            response = requests.put(manifest_upload_url, data=f.read())
+        if response.status_code not in [200, 204]:
+            raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
+    except Exception as e:
+        console.print("[red]Error:[/red] Failed to upload manifest.json")
+        console.print(f"Reason: {e}")
+        sys.exit(4)
+
+    # Upload catalog.json
+    console.print(f'Uploading catalog from path "{catalog_path}"')
+    try:
+        with open(catalog_path, "rb") as f:
+            response = requests.put(catalog_upload_url, data=f.read())
+        if response.status_code not in [200, 204]:
+            raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
+    except Exception as e:
+        console.print("[red]Error:[/red] Failed to upload catalog.json")
+        console.print(f"Reason: {e}")
+        sys.exit(4)
+
+    # Notify upload completion
+    console.print("Notifying upload completion...")
+    try:
+        client.upload_completed(session_id=session_id, commit_sha=ci_info.commit_sha)
+    except RecceCloudException as e:
+        console.print("[yellow]Warning:[/yellow] Failed to notify upload completion")
+        console.print(f"Reason: {e.reason}")
+        # Non-fatal, continue
+    except Exception as e:
+        console.print("[yellow]Warning:[/yellow] Failed to notify upload completion")
+        console.print(f"Reason: {e}")
+        # Non-fatal, continue
+
+    # Success!
+    console.rule("Uploaded Successfully", style="green")
+    console.print(f'Uploaded dbt artifacts to Recce Cloud for session ID "{session_id}"')
+    console.print(f'Artifacts from: "{os.path.abspath(target_path)}"')
+
+    if ci_info.cr_url:
+        console.print(f"Change request: {ci_info.cr_url}")
+
+    sys.exit(0)

package/recce-source/tests/adapter/dbt_adapter/conftest.py

@@ -0,0 +1,17 @@
+from unittest.mock import patch
+
+import pytest
+
+from recce.core import set_default_context
+
+from .dbt_test_helper import DbtTestHelper
+
+
+@pytest.fixture
+def dbt_test_helper():
+    with patch("recce.adapter.dbt_adapter.log_performance"):
+        helper = DbtTestHelper()
+        context = helper.context
+        set_default_context(context)
+        yield helper
+        helper.cleanup()

package/recce-source/tests/adapter/dbt_adapter/dbt_test_helper.py

@@ -0,0 +1,298 @@
+import os
+import textwrap
+import uuid
+from datetime import datetime
+from io import StringIO
+
+from dbt.contracts.graph.nodes import (
+    ModelNode,
+    SeedNode,
+    SnapshotNode,
+    SourceDefinition,
+)
+from dbt.contracts.results import (
+    CatalogArtifact,
+    CatalogTable,
+    ColumnMetadata,
+    TableMetadata,
+)
+
+from recce.adapter.dbt_adapter import DbtAdapter, as_manifest, load_manifest
+from recce.core import RecceContext
+
+
+class DbtTestHelper:
+
+    def __init__(self):
+        schema_prefix = "schema_" + uuid.uuid4().hex
+        self.base_schema = f"{schema_prefix}_base"
+        self.curr_schema = f"{schema_prefix}_curr"
+
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        project_dir = os.path.join(current_dir, "test_proj")
+        profiles_dir = project_dir
+        manifest_path = os.path.join(project_dir, "manifest.json")
+
+        dbt_adapter = DbtAdapter.load(
+            no_artifacts=True,
+            project_dir=project_dir,
+            profiles_dir=profiles_dir,
+        )
+
+        context = RecceContext()
+        context.adapter_type = "dbt"
+        context.adapter = dbt_adapter
+        context.schema_prefix = schema_prefix
+        self.adapter = dbt_adapter
+        self.context = context
+        curr_writable_manifest = load_manifest(manifest_path)
+        base_writable_manifest = load_manifest(manifest_path)
+
+        self.curr_manifest = as_manifest(curr_writable_manifest)
+        self.base_manifest = as_manifest(base_writable_manifest)
+        now = datetime.now()
+        self.curr_catalog = CatalogArtifact.from_results(
+            generated_at=now,
+            nodes={},
+            sources={},
+            compile_results=None,
+            errors=None,
+        )
+        self.base_catalog = CatalogArtifact.from_results(
+            generated_at=now,
+            nodes={},
+            sources={},
+            compile_results=None,
+            errors=None,
+        )
+        self.context = context
+
+        self.adapter.execute(f"CREATE schema IF NOT EXISTS {self.base_schema}")
+        self.adapter.execute(f"CREATE schema IF NOT EXISTS {self.curr_schema}")
+        self.adapter.set_artifacts(
+            base_writable_manifest,
+            curr_writable_manifest,
+            self.curr_manifest,
+            self.base_manifest,
+            self.base_catalog,
+            self.curr_catalog,
+        )
+
+    def create_model(
+        self,
+        model_name,
+        base_csv=None,
+        curr_csv=None,
+        base_sql=None,
+        curr_sql=None,
+        depends_on=None,
+        disabled=False,
+        unique_id=None,
+        resource_type="model",
+        package_name="recce_test",
+        base_columns: dict[str, str] = None,
+        curr_columns: dict[str, str] = None,
+        patch_func=None,
+    ):
+        # unique_id = f"model.{package_name}.{model_name}"
+        unique_id = unique_id if unique_id else model_name
+        if depends_on is None:
+            depends_on = []
+
+        def _add_model_to_manifest(base):
+            if base:
+                schema = self.base_schema
+                manifest = self.base_manifest
+                catalog = self.base_catalog
+                csv = base_csv
+                sql = base_sql
+                columns = base_columns
+            else:
+                schema = self.curr_schema
+                manifest = self.curr_manifest
+                catalog = self.curr_catalog
+                csv = curr_csv
+                sql = curr_sql
+                columns = curr_columns
+
+            if csv:
+                dbt_adapter = self.adapter
+                csv = textwrap.dedent(csv)
+                with dbt_adapter.connection_named("create model"):
+                    import pandas as pd
+
+                    df = pd.read_csv(StringIO(csv))  # noqa: F841
+                    # DuckDB disables python_scan_all_frames by default after 1.1,
+                    # ref: https://github.com/duckdb/duckdb/pull/13896
+                    dbt_adapter.execute("SET python_scan_all_frames=true")
+                    dbt_adapter.execute(f"CREATE TABLE {schema}.{model_name} AS SELECT * FROM df")
+            raw_code = sql if sql else csv
+
+            if columns:
+                index = 1
+                table = CatalogTable(TableMetadata(type="BASE TABLE", schema=schema, name=model_name), {}, {})
+                catalog.nodes[unique_id] = table
+                for column, column_type in columns.items():
+                    col_data = ColumnMetadata(type=column_type, index=index, name=column)
+                    catalog.nodes[unique_id].columns[column] = col_data
+                    index = index + 1
+
+            node_dict = {
+                "resource_type": resource_type,
+                "name": model_name,
+                "package_name": package_name,
+                "path": "",
+                "original_file_path": "",
+                "unique_id": unique_id,
+                "fqn": [
+                    package_name,
+                    model_name,
+                ],
+                "schema": schema,
+                "alias": model_name,
+                "checksum": {
+                    "name": "sha256",
+                    "checksum": hash(raw_code),
+                },
+                "raw_code": raw_code,
+                "config": {
+                    "materialized": "table",
+                    "tags": ["test_tag"],
+                },
+                "tags": ["test_tag"],
+                "depends_on": {"nodes": depends_on},
+            }
+            if patch_func:
+                patch_func(node_dict)
+
+            if resource_type == "snapshot":
+                node = SnapshotNode.from_dict(node_dict)
+            elif resource_type == "seed":
+                node = SeedNode.from_dict(node_dict)
+            else:
+                node = ModelNode.from_dict(node_dict)
+
+            if disabled:
+                manifest.add_disabled_nofile(node)
+            else:
+                manifest.add_node_nofile(node)
+            return node
+
+        if base_csv or base_sql:
+            _add_model_to_manifest(True)
+
+        if curr_csv or curr_sql:
+            _add_model_to_manifest(False)
+
+        self.adapter.set_artifacts(
+            self.base_manifest.writable_manifest(),
+            self.curr_manifest.writable_manifest(),
+            self.curr_manifest,
+            self.base_manifest,
+            self.base_catalog,
+            self.curr_catalog,
+        )
+
+    def remove_model(self, model_name):
+        dbt_adapter = self.adapter
+        with dbt_adapter.connection_named("cleanup"):
+            dbt_adapter.execute(f"DROP TABLE IF EXISTS {self.base_schema}.{model_name}")
+            dbt_adapter.execute(f"DROP TABLE IF EXISTS {self.curr_schema}.{model_name} ")
+
+    def create_source(
+        self,
+        source_name,
+        table_name,
+        base_csv=None,
+        curr_csv=None,
+        unique_id=None,
+        package_name="recce_test",
+        base_columns: dict[str, str] = None,
+        curr_columns: dict[str, str] = None,
+        patch_func=None,
+    ):
+        unique_id = unique_id if unique_id else f"source.{package_name}.{source_name}.{table_name}"
+
+        def _add_source_to_manifest(base):
+            if base:
+                schema = self.base_schema
+                manifest = self.base_manifest
+                catalog = self.base_catalog
+                csv = base_csv
+                columns = base_columns
+            else:
+                schema = self.curr_schema
+                manifest = self.curr_manifest
+                catalog = self.curr_catalog
+                csv = curr_csv
+                columns = curr_columns
+
+            if csv:
+                dbt_adapter = self.adapter
+                csv = textwrap.dedent(csv)
+                with dbt_adapter.connection_named("create source"):
+                    import pandas as pd
+
+                    df = pd.read_csv(StringIO(csv))  # noqa: F841
+                    # DuckDB disables python_scan_all_frames by default after 1.1,
+                    # ref: https://github.com/duckdb/duckdb/pull/13896
+                    dbt_adapter.execute("SET python_scan_all_frames=true")
+                    dbt_adapter.execute(f"CREATE TABLE {schema}.{table_name} AS SELECT * FROM df")
+
+            if columns:
+                index = 1
+                table = CatalogTable(TableMetadata(type="BASE TABLE", schema=schema, name=table_name), {}, {})
+                catalog.sources[unique_id] = table
+                for column, column_type in columns.items():
+                    col_data = ColumnMetadata(type=column_type, index=index, name=column)
+                    catalog.sources[unique_id].columns[column] = col_data
+                    index = index + 1
+
+            node_dict = {
+                "resource_type": "source",
+                "source_name": source_name,
+                "name": table_name,
+                "identifier": table_name,
+                "source_description": "test source",
+                "loader": "",
+                "package_name": package_name,
+                "path": "",
+                "original_file_path": "",
+                "unique_id": unique_id,
+                "fqn": [
+                    package_name,
+                    source_name,
+                    table_name,
+                ],
+                "schema": schema,
+            }
+            if patch_func:
+                patch_func(node_dict)
+
+            source = SourceDefinition.from_dict(node_dict)
+            manifest.sources[unique_id] = source
+            return source
+
+        if base_csv:
+            _add_source_to_manifest(True)
+
+        if curr_csv:
+            _add_source_to_manifest(False)
+
+        self.adapter.set_artifacts(
+            self.base_manifest.writable_manifest(),
+            self.curr_manifest.writable_manifest(),
+            self.curr_manifest,
+            self.base_manifest,
+            self.base_catalog,
+            self.curr_catalog,
+        )
+
+    def cleanup(self):
+        dbt_adapter = self.adapter
+        with dbt_adapter.connection_named("cleanup"):
+            dbt_adapter.execute(f"DROP SCHEMA IF EXISTS {self.base_schema} CASCADE")
+            dbt_adapter.execute(f"DROP SCHEMA IF EXISTS {self.curr_schema} CASCADE")
+
+    def create_snapshot(self, sanpshot_name, base_csv, curr_csv, depends_on=[]):
+        self.create_model(sanpshot_name, base_csv, curr_csv, depends_on=depends_on, resource_type="snapshot")

package/recce-source/tests/adapter/dbt_adapter/test_dbt_adapter.py

@@ -0,0 +1,25 @@
+from recce.adapter.dbt_adapter import DbtAdapter, dbt_supported_registry
+
+
+def test_dbt_adapter_support_tasks(dbt_test_helper):
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+
+    # Test dbt task support
+    support_tasks = adapter.support_tasks()
+
+    for task_type in dbt_supported_registry:
+        task = task_type.value
+        assert task in support_tasks
+        assert support_tasks[task] is True
+
+
+def test_dbt_adapter_support_tasks_without_required_dbt_package(dbt_test_helper):
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+    # Mock the macros in manifest to simulate no required dbt package installed
+    adapter.manifest.macros = {}
+
+    support_tasks = adapter.support_tasks()
+
+    for task_type in dbt_supported_registry:
+        task = task_type.value
+        assert task in support_tasks