@datarecce/ui 0.1.29 → 0.1.31

package/recce-source/tests/tasks/test_schema.py

@@ -0,0 +1,122 @@
+import os
+from unittest import TestCase
+from unittest.mock import MagicMock
+
+import pytest
+
+from recce.adapter.dbt_adapter import DbtAdapter, load_catalog, load_manifest
+from recce.core import RecceContext, set_default_context
+from recce.run import schema_diff_should_be_approved
+
+
+def test_validator():
+    from recce.tasks.schema import SchemaDiffCheckValidator
+
+    validator = SchemaDiffCheckValidator()
+
+    def validate(params: dict):
+        validator.validate(
+            {
+                "name": "test",
+                "type": "schema_diff",
+                "params": params,
+            }
+        )
+
+    # Select all models
+    validate({})
+
+    # Select by node name
+    validate(
+        {
+            "node_id": "abc",
+        }
+    )
+    validate(
+        {
+            "node_id": ["abc"],
+        }
+    )
+
+    # Select by selector
+    validate(
+        {
+            "select": "customers",
+            "exclude": "customers",
+            "packages": ["jaffle_shop"],
+            "view_mode": "all",
+        }
+    )
+
+    # packages should be an array
+    with pytest.raises(ValueError):
+        validate(
+            {
+                "packages": "jaffle_shop",
+            }
+        )
+
+    # view_mode should be 'all' or 'changed_models'
+    validate(
+        {
+            "view_mode": None,
+        }
+    )
+    validate(
+        {
+            "view_mode": "all",
+        }
+    )
+    with pytest.raises(ValueError):
+        validate(
+            {
+                "view_mode": "abc",
+            }
+        )
+
+
+test_root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+class TestSchemaDiffAutoApprove(TestCase):
+
+    def setUp(self):
+        self.default_context = MagicMock(spec=RecceContext)
+        manifest = load_manifest(path=os.path.join(test_root_path, "manifest.json"))
+        catalog = load_catalog(path=os.path.join(test_root_path, "catalog.json"))
+        dbt_adapter = DbtAdapter(curr_manifest=manifest, curr_catalog=catalog)
+        self.default_context.adapter = dbt_adapter
+
+        dbt_adapter.adapter = MagicMock()
+        dbt_adapter.adapter.type.return_value = None
+
+        dbt_adapter.select_nodes = MagicMock()
+        # Base and Current will be the same
+        self.default_context.get_lineage.return_value = dbt_adapter.get_lineage()
+        set_default_context(self.default_context)
+
+    def test_schema_diff_should_be_approved(self):
+        # Node_id is string
+        is_approved = schema_diff_should_be_approved(
+            {
+                "node_id": "model.jaffle_shop.customers",
+            }
+        )
+        assert is_approved is True
+
+        # Node_id is list
+        is_approved = schema_diff_should_be_approved(
+            {
+                "node_id": ["model.jaffle_shop.customers"],
+            }
+        )
+        assert is_approved is True
+
+        # Select all models
+        self.default_context.adapter.select_nodes.return_value = ["model.jaffle_shop.customers"]
+        is_approved = schema_diff_should_be_approved(
+            {
+                "select": "customers",
+            }
+        )
+        assert is_approved is True

package/recce-source/tests/tasks/test_top_k.py

@@ -0,0 +1,77 @@
+import pytest
+
+from recce.tasks import TopKDiffTask
+from recce.tasks.top_k import TopKDiffCheckValidator
+
+
+def test_top_k(dbt_test_helper):
+    csv_data_curr = """
+        customer_id,name,age
+        1,Alice,30
+        2,Bob,25
+        3,Charlie,35
+        4,Bob,35
+        """
+
+    csv_data_base = """
+        customer_id,name,age
+        1,Alice,35
+        2,Bob,25
+        3,Charlie,35
+        4,,35
+        """
+
+    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
+
+    params = dict(
+        model="customers",
+        column_name="name",
+        k=50,
+    )
+
+    task = TopKDiffTask(params)
+    run_result = task.execute()
+
+    # {
+    #     'values': ['Bob', 'Alice', 'Charlie'],
+    #     'counts': [2, 1, 1],
+    #     'valids': 4,
+    #     'total': 4
+    # }
+    assert run_result["current"]["values"][0] == "Bob"
+    assert run_result["current"]["counts"][0] == 2
+    assert run_result["current"]["valids"] == 4
+    assert run_result["current"]["total"] == 4
+
+    # {
+    #     'values': ['Bob', 'Alice', 'Charlie'],
+    #     'counts': [1, 1, 1],
+    #     'valids': 3,
+    #     'total': 4
+    # }
+    assert run_result["base"]["counts"][0] == 1
+    assert run_result["base"]["valids"] == 3
+    assert run_result["base"]["total"] == 4
+
+
+def test_validator():
+    def validate(params: dict = {}, view_options: dict = {}):
+        TopKDiffCheckValidator().validate(
+            {
+                "name": "test",
+                "type": "top_k_diff",
+                "params": params,
+                "view_options": view_options,
+            }
+        )
+
+    validate(
+        {
+            "model": "customers",
+            "column_name": "name",
+            "k": 50,
+        }
+    )
+
+    with pytest.raises(ValueError):
+        validate({})

package/recce-source/tests/tasks/test_utils.py

@@ -0,0 +1,439 @@
+"""Tests for task utility functions."""
+
+import unittest
+
+from recce.tasks.utils import normalize_keys_to_columns, strip_identifier_quotes
+
+
+class TestStripIdentifierQuotes(unittest.TestCase):
+    """Tests for strip_identifier_quotes function."""
+
+    def test_double_quotes(self):
+        """Double quotes (PostgreSQL, Snowflake) are stripped."""
+        assert strip_identifier_quotes('"myColumn"') == "myColumn"
+        assert strip_identifier_quotes('"customerID"') == "customerID"
+
+    def test_backticks(self):
+        """Backticks (MySQL, BigQuery) are stripped."""
+        assert strip_identifier_quotes("`my_column`") == "my_column"
+        assert strip_identifier_quotes("`Column Name`") == "Column Name"
+
+    def test_square_brackets(self):
+        """Square brackets (SQL Server) are stripped."""
+        assert strip_identifier_quotes("[Column Name]") == "Column Name"
+        assert strip_identifier_quotes("[my_column]") == "my_column"
+
+    def test_unquoted_passthrough(self):
+        """Unquoted identifiers pass through unchanged."""
+        assert strip_identifier_quotes("regular_column") == "regular_column"
+        assert strip_identifier_quotes("myColumn") == "myColumn"
+        assert strip_identifier_quotes("PAYMENT_ID") == "PAYMENT_ID"
+
+    def test_empty_and_short_strings(self):
+        """Empty and short strings pass through unchanged."""
+        assert strip_identifier_quotes("") == ""
+        assert strip_identifier_quotes("a") == "a"
+        assert strip_identifier_quotes('"') == '"'  # Single quote char, not a pair
+
+    def test_mismatched_quotes(self):
+        """Mismatched quotes are not stripped."""
+        assert strip_identifier_quotes('"column`') == '"column`'
+        assert strip_identifier_quotes('`column"') == '`column"'
+        assert strip_identifier_quotes('"column]') == '"column]'
+
+
+class TestNormalizeKeysToColumns(unittest.TestCase):
+    """Tests for normalize_keys_to_columns function."""
+
+    # ========================================================================
+    # Warehouse-specific casing tests
+    # ========================================================================
+
+    def test_snowflake_uppercase(self):
+        """Snowflake returns UPPERCASE column names."""
+        keys = ["payment_id", "order_id"]
+        columns = ["PAYMENT_ID", "ORDER_ID", "AMOUNT"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["PAYMENT_ID", "ORDER_ID"]
+
+    def test_postgres_lowercase(self):
+        """PostgreSQL/Redshift returns lowercase column names."""
+        keys = ["Payment_ID", "Order_ID"]
+        columns = ["payment_id", "order_id", "amount"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["payment_id", "order_id"]
+
+    def test_bigquery_preserves_case(self):
+        """BigQuery preserves original case."""
+        keys = ["PaymentId", "OrderId"]
+        columns = ["PaymentId", "OrderId", "Amount"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["PaymentId", "OrderId"]
+
+    def test_mixed_case_matching(self):
+        """Mixed case input matches correctly via case-insensitive fallback."""
+        keys = ["PAYMENT_id", "order_ID"]
+        columns = ["Payment_Id", "Order_Id", "Amount"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["Payment_Id", "Order_Id"]
+
+    # ========================================================================
+    # Quoted column tests (exact match first)
+    # ========================================================================
+
+    def test_quoted_column_exact_match(self):
+        """Quoted columns with exact match are preserved."""
+        keys = ["preCommitID"]
+        columns = ["preCommitID", "order_id", "amount"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["preCommitID"]
+
+    def test_quoted_column_exact_match_mixed_with_unquoted(self):
+        """Mix of quoted (exact match) and unquoted (case-insensitive) columns."""
+        keys = ["preCommitID", "ORDER_ID"]
+        columns = ["preCommitID", "order_id", "amount"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["preCommitID", "order_id"]
+
+    def test_quoted_column_case_insensitive_fallback(self):
+        """
+        If quoted column doesn't match exactly, falls back to case-insensitive.
+        Note: This is a last-ditch fallback to capture potential mistakes the user
+        made when typing their primary key. It is a fallback measure, not meant to
+        be a fully robust solution.
+        """
+        keys = ["precommitid"]  # User provides lowercase
+        columns = ["preCommitID", "order_id"]  # But warehouse has mixed case
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["preCommitID"]  # Falls back to case-insensitive match
+
+    def test_multiple_quoted_columns(self):
+        """Multiple quoted columns with preserved casing."""
+        keys = ["preCommitID", "postCommitID", "userName"]
+        columns = ["preCommitID", "postCommitID", "userName", "created_at"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["preCommitID", "postCommitID", "userName"]
+
+    # ========================================================================
+    # SQL identifier quote stripping tests
+    # ========================================================================
+
+    def test_double_quoted_key_stripped_and_matched(self):
+        """Double-quoted keys are stripped before matching (PostgreSQL, Snowflake)."""
+        keys = ['"customerID"']
+        columns = ["customerID", "amount", "created_at"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["customerID"]
+
+    def test_backtick_quoted_key_stripped_and_matched(self):
+        """Backtick-quoted keys are stripped before matching (MySQL, BigQuery)."""
+        keys = ["`my_column`"]
+        columns = ["MY_COLUMN", "other_col"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["MY_COLUMN"]  # Case-insensitive match after stripping
+
+    def test_square_bracket_quoted_key_stripped_and_matched(self):
+        """Square bracket-quoted keys are stripped before matching (SQL Server)."""
+        keys = ["[Column Name]"]
+        columns = ["Column Name", "other_col"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["Column Name"]
+
+    def test_quoted_key_empty_columns_returns_unquoted(self):
+        """Quoted keys with empty columns list returns unquoted keys."""
+        keys = ['"customerID"', "`another_col`"]
+
+        result = normalize_keys_to_columns(keys, [])
+
+        assert result == ["customerID", "another_col"]
+
+    def test_mixed_quoted_and_unquoted_keys(self):
+        """Mix of quoted and unquoted keys are handled correctly."""
+        keys = ['"customerID"', "order_id", "`created_at`"]
+        columns = ["customerID", "ORDER_ID", "CREATED_AT"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["customerID", "ORDER_ID", "CREATED_AT"]
+
+    def test_quoted_key_case_insensitive_match(self):
+        """Quoted key after stripping matches case-insensitively."""
+        keys = ['"my_column"']  # Lowercase inside quotes
+        columns = ["MY_COLUMN", "other"]  # Uppercase in warehouse (Snowflake)
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["MY_COLUMN"]
+
+    # ========================================================================
+    # Edge cases and null handling
+    # ========================================================================
+
+    def test_none_keys_returns_none(self):
+        """None keys input returns None."""
+        result = normalize_keys_to_columns(None, ["col1", "col2"])
+
+        assert result is None
+
+    def test_empty_keys_returns_empty(self):
+        """Empty keys list returns empty list."""
+        result = normalize_keys_to_columns([], ["col1", "col2"])
+
+        assert result == []
+
+    def test_empty_columns_preserves_keys(self):
+        """Empty columns list preserves original keys."""
+        keys = ["payment_id", "order_id"]
+
+        result = normalize_keys_to_columns(keys, [])
+
+        assert result == ["payment_id", "order_id"]
+
+    def test_unmatched_key_preserved(self):
+        """Keys not found in columns are preserved as-is."""
+        keys = ["payment_id", "nonexistent_col"]
+        columns = ["PAYMENT_ID", "ORDER_ID"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["PAYMENT_ID", "nonexistent_col"]
+
+    def test_single_key(self):
+        """Single key normalization works."""
+        result = normalize_keys_to_columns(["id"], ["ID", "NAME"])
+
+        assert result == ["ID"]
+
+    def test_special_columns_in_a_in_b(self):
+        """IN_A and IN_B columns normalize correctly."""
+        keys = ["in_a", "in_b"]
+        columns = ["ID", "IN_A", "IN_B"]
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        assert result == ["IN_A", "IN_B"]
+
+    # ========================================================================
+    # Case collision edge cases
+    # ========================================================================
+
+    def test_exact_match_takes_priority_over_case_insensitive(self):
+        """When exact match exists, it takes priority even if case-insensitive would match differently."""
+        # This tests the scenario where there might be columns that differ only by case
+        keys = ["ID"]
+        columns = ["ID", "id"]  # Both exist (rare but possible with quoting)
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        # Should use exact match "ID", not fall back to case-insensitive
+        assert result == ["ID"]
+
+    def test_case_insensitive_when_no_exact_match(self):
+        """Case-insensitive matching when exact match doesn't exist."""
+        keys = ["Id"]  # Mixed case, won't match exactly
+        columns = ["ID", "id"]  # Both exist
+
+        result = normalize_keys_to_columns(keys, columns)
+
+        # Falls back to case-insensitive, last one in the map wins
+        # (This is acceptable - having same-name columns with different case is an edge case)
+        assert result[0] in ["ID", "id"]
+
+    # Add to tests/tasks/test_utils.py
+
+
+class TestNormalizeBooleanFlagColumns(unittest.TestCase):
+    """Tests for normalize_boolean_flag_columns function."""
+
+    def test_normalizes_uppercase_in_a_in_b(self):
+        """Snowflake returns UPPERCASE - should normalize to lowercase."""
+        from recce.tasks.dataframe import (
+            DataFrame,
+            DataFrameColumn,
+            DataFrameColumnType,
+        )
+        from recce.tasks.utils import normalize_boolean_flag_columns
+
+        df = DataFrame(
+            columns=[
+                DataFrameColumn(key="ID", name="ID", type=DataFrameColumnType.INTEGER),
+                DataFrameColumn(key="IN_A", name="IN_A", type=DataFrameColumnType.BOOLEAN),
+                DataFrameColumn(key="IN_B", name="IN_B", type=DataFrameColumnType.BOOLEAN),
+                DataFrameColumn(key="VALUE", name="VALUE", type=DataFrameColumnType.NUMBER),
+            ],
+            data=[(1, True, True, 100), (2, True, False, 200)],
+        )
+
+        result = normalize_boolean_flag_columns(df)
+
+        column_keys = [col.key for col in result.columns]
+        self.assertEqual(column_keys, ["ID", "in_a", "in_b", "VALUE"])
+
+        column_names = [col.name for col in result.columns]
+        self.assertEqual(column_names, ["ID", "in_a", "in_b", "VALUE"])
+
+    def test_preserves_lowercase_in_a_in_b(self):
+        """PostgreSQL returns lowercase - should remain unchanged."""
+        from recce.tasks.dataframe import (
+            DataFrame,
+            DataFrameColumn,
+            DataFrameColumnType,
+        )
+        from recce.tasks.utils import normalize_boolean_flag_columns
+
+        df = DataFrame(
+            columns=[
+                DataFrameColumn(key="id", name="id", type=DataFrameColumnType.INTEGER),
+                DataFrameColumn(key="in_a", name="in_a", type=DataFrameColumnType.BOOLEAN),
+                DataFrameColumn(key="in_b", name="in_b", type=DataFrameColumnType.BOOLEAN),
+                DataFrameColumn(key="value", name="value", type=DataFrameColumnType.NUMBER),
+            ],
+            data=[(1, True, True, 100)],
+        )
+
+        result = normalize_boolean_flag_columns(df)
+
+        column_keys = [col.key for col in result.columns]
+        self.assertEqual(column_keys, ["id", "in_a", "in_b", "value"])
+
+    def test_handles_mixed_case_in_a_in_b(self):
+        """Mixed case like In_A should normalize to lowercase."""
+        from recce.tasks.dataframe import (
+            DataFrame,
+            DataFrameColumn,
+            DataFrameColumnType,
+        )
+        from recce.tasks.utils import normalize_boolean_flag_columns
+
+        df = DataFrame(
+            columns=[
+                DataFrameColumn(key="id", name="id", type=DataFrameColumnType.INTEGER),
+                DataFrameColumn(key="In_A", name="In_A", type=DataFrameColumnType.BOOLEAN),
+                DataFrameColumn(key="In_B", name="In_B", type=DataFrameColumnType.BOOLEAN),
+            ],
+            data=[(1, True, False)],
+        )
+
+        result = normalize_boolean_flag_columns(df)
+
+        column_keys = [col.key for col in result.columns]
+        self.assertEqual(column_keys, ["id", "in_a", "in_b"])
+
+    def test_preserves_other_columns(self):
+        """Non in_a/in_b columns should remain unchanged."""
+        from recce.tasks.dataframe import (
+            DataFrame,
+            DataFrameColumn,
+            DataFrameColumnType,
+        )
+        from recce.tasks.utils import normalize_boolean_flag_columns
+
+        df = DataFrame(
+            columns=[
+                DataFrameColumn(key="USER_ID", name="USER_ID", type=DataFrameColumnType.INTEGER),
+                DataFrameColumn(key="IN_A", name="IN_A", type=DataFrameColumnType.BOOLEAN),
+                DataFrameColumn(key="IN_B", name="IN_B", type=DataFrameColumnType.BOOLEAN),
+                DataFrameColumn(key="base__VALUE", name="base__VALUE", type=DataFrameColumnType.NUMBER),
+                DataFrameColumn(key="current__VALUE", name="current__VALUE", type=DataFrameColumnType.NUMBER),
+            ],
+            data=[(1, True, True, 100, 150)],
+        )
+
+        result = normalize_boolean_flag_columns(df)
+
+        column_keys = [col.key for col in result.columns]
+        # Only in_a/in_b should be lowercased
+        self.assertEqual(column_keys, ["USER_ID", "in_a", "in_b", "base__VALUE", "current__VALUE"])
+
+    def test_preserves_data(self):
+        """Data should remain unchanged."""
+        from recce.tasks.dataframe import (
+            DataFrame,
+            DataFrameColumn,
+            DataFrameColumnType,
+        )
+        from recce.tasks.utils import normalize_boolean_flag_columns
+
+        original_data = [(1, True, False, 100), (2, False, True, 200)]
+        df = DataFrame(
+            columns=[
+                DataFrameColumn(key="ID", name="ID", type=DataFrameColumnType.INTEGER),
+                DataFrameColumn(key="IN_A", name="IN_A", type=DataFrameColumnType.BOOLEAN),
+                DataFrameColumn(key="IN_B", name="IN_B", type=DataFrameColumnType.BOOLEAN),
+                DataFrameColumn(key="VALUE", name="VALUE", type=DataFrameColumnType.NUMBER),
+            ],
+            data=original_data,
+        )
+
+        result = normalize_boolean_flag_columns(df)
+
+        self.assertEqual(result.data, original_data)
+
+    def test_preserves_limit_and_more(self):
+        """limit and more fields should be preserved."""
+        from recce.tasks.dataframe import (
+            DataFrame,
+            DataFrameColumn,
+            DataFrameColumnType,
+        )
+        from recce.tasks.utils import normalize_boolean_flag_columns
+
+        df = DataFrame(
+            columns=[
+                DataFrameColumn(key="id", name="id", type=DataFrameColumnType.INTEGER),
+                DataFrameColumn(key="IN_A", name="IN_A", type=DataFrameColumnType.BOOLEAN),
+                DataFrameColumn(key="IN_B", name="IN_B", type=DataFrameColumnType.BOOLEAN),
+            ],
+            data=[(1, True, True)],
+            limit=100,
+            more=True,
+        )
+
+        result = normalize_boolean_flag_columns(df)
+
+        self.assertEqual(result.limit, 100)
+        self.assertEqual(result.more, True)
+
+    def test_handles_dataframe_without_in_columns(self):
+        """DataFrames without in_a/in_b should pass through unchanged."""
+        from recce.tasks.dataframe import (
+            DataFrame,
+            DataFrameColumn,
+            DataFrameColumnType,
+        )
+        from recce.tasks.utils import normalize_boolean_flag_columns
+
+        df = DataFrame(
+            columns=[
+                DataFrameColumn(key="id", name="id", type=DataFrameColumnType.INTEGER),
+                DataFrameColumn(key="name", name="name", type=DataFrameColumnType.TEXT),
+            ],
+            data=[(1, "Alice"), (2, "Bob")],
+        )
+
+        result = normalize_boolean_flag_columns(df)
+
+        column_keys = [col.key for col in result.columns]
+        self.assertEqual(column_keys, ["id", "name"])