@datarecce/ui 0.1.41 → 0.2.0

package/recce-source/tests/tasks/test_query.py

@@ -1,528 +0,0 @@
-import pytest
-
-from recce.tasks import QueryDiffTask, QueryTask
-
-# =============================================================================
-# QueryTask Tests (single environment query)
-# =============================================================================
-
-
-def test_query(dbt_test_helper):
-    """Basic QueryTask test - executes query on current environment."""
-    csv_data = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers", csv_data, csv_data)
-    params = {"sql_template": 'select * from {{ ref("customers") }}'}
-    task = QueryTask(params)
-    run_result = task.execute()
-    assert len(run_result.data) == 3
-
-
-# =============================================================================
-# _query_diff Tests (no primary_keys - client-side diff)
-# =============================================================================
-
-
-def test_query_diff_in_client(dbt_test_helper):
-    """Basic _query_diff test - returns base and current DataFrames."""
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("customers") }}'}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    assert len(run_result.base.data) == 3
-    assert len(run_result.current.data) == 3
-    # No diff result when no primary_keys
-    assert run_result.diff is None
-
-
-def test_query_diff_in_client_with_base_sql_template(dbt_test_helper):
-    """Test _query_diff with different SQL for base vs current."""
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = {
-        "base_sql_template": 'select * from {{ ref("customers") }} where customer_id <= 2',
-        "sql_template": 'select * from {{ ref("customers") }}',
-    }
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    assert len(run_result.base.data) == 2
-    assert len(run_result.current.data) == 3
-
-
-def test_query_diff_in_client_identical_data(dbt_test_helper):
-    """Test _query_diff when base and current are identical."""
-    csv_data = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        """
-
-    dbt_test_helper.create_model("identical", csv_data, csv_data)
-    params = {"sql_template": 'select * from {{ ref("identical") }}'}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    assert len(run_result.base.data) == 2
-    assert len(run_result.current.data) == 2
-    # Data should be the same
-    assert run_result.base.data == run_result.current.data
-
-
-def test_query_diff_in_client_empty_base(dbt_test_helper):
-    """Test _query_diff when base returns no rows."""
-    csv_data_base = """
-        customer_id,name,age
-        """
-
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        """
-
-    dbt_test_helper.create_model("empty_base", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("empty_base") }}'}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    assert len(run_result.base.data) == 0
-    assert len(run_result.current.data) == 2
-
-
-def test_query_diff_in_client_empty_current(dbt_test_helper):
-    """Test _query_diff when current returns no rows."""
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        """
-
-    csv_data_curr = """
-        customer_id,name,age
-        """
-
-    dbt_test_helper.create_model("empty_curr", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("empty_curr") }}'}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    assert len(run_result.base.data) == 2
-    assert len(run_result.current.data) == 0
-
-
-# =============================================================================
-# _query_diff_join Tests (with primary_keys - warehouse-side diff)
-# =============================================================================
-
-
-def test_query_diff_in_warehouse(dbt_test_helper):
-    """Basic _query_diff_join test - returns diff DataFrame with in_a/in_b flags."""
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("customers") }}', "primary_keys": ["customer_id"]}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    # Should return diff (not base/current)
-    assert run_result.diff is not None
-    assert run_result.base is None
-    assert run_result.current is None
-    # One row changed (customer_id=1, age: 35->30), appears twice (old and new)
-    assert len(run_result.diff.data) == 2
-
-
-def test_query_diff_in_warehouse_with_base_sql_template(dbt_test_helper):
-    """Test _query_diff_join with different SQL for base vs current."""
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = {
-        "base_sql_template": 'select * from {{ ref("customers") }} where customer_id == 1',
-        "sql_template": 'select * from {{ ref("customers") }}',
-        "primary_keys": ["customer_id"],
-    }
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    # Base has 1 row (customer_id=1), current has 3 rows
-    # Diff should show: customer_id=1 modified (2 rows), customer_id=2,3 added (2 rows)
-    assert len(run_result.diff.data) == 4
-
-
-def test_query_diff_in_warehouse_identical_data(dbt_test_helper):
-    """Test _query_diff_join when base and current are identical."""
-    csv_data = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("identical_wh", csv_data, csv_data)
-    params = {"sql_template": 'select * from {{ ref("identical_wh") }}', "primary_keys": ["customer_id"]}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    # No differences, diff should be empty
-    assert len(run_result.diff.data) == 0
-
-
-def test_query_diff_in_warehouse_added_rows(dbt_test_helper):
-    """Test _query_diff_join detecting rows added in current."""
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        """
-
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("added_rows", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("added_rows") }}', "primary_keys": ["customer_id"]}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    # One row added (customer_id=3)
-    assert len(run_result.diff.data) == 1
-
-    # Check the added row has in_a=False, in_b=True
-    row = run_result.diff.data[0]
-    in_a_idx = next(i for i, c in enumerate(run_result.diff.columns) if c.key.lower() == "in_a")
-    in_b_idx = next(i for i, c in enumerate(run_result.diff.columns) if c.key.lower() == "in_b")
-    assert row[in_a_idx] is False
-    assert row[in_b_idx] is True
-
-
-def test_query_diff_in_warehouse_removed_rows(dbt_test_helper):
-    """Test _query_diff_join detecting rows removed in current."""
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        """
-
-    dbt_test_helper.create_model("removed_rows", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("removed_rows") }}', "primary_keys": ["customer_id"]}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    # One row removed (customer_id=3)
-    assert len(run_result.diff.data) == 1
-
-    # Check the removed row has in_a=True, in_b=False
-    row = run_result.diff.data[0]
-    in_a_idx = next(i for i, c in enumerate(run_result.diff.columns) if c.key.lower() == "in_a")
-    in_b_idx = next(i for i, c in enumerate(run_result.diff.columns) if c.key.lower() == "in_b")
-    assert row[in_a_idx] is True
-    assert row[in_b_idx] is False
-
-
-def test_query_diff_in_warehouse_modified_rows(dbt_test_helper):
-    """Test _query_diff_join detecting modified rows (same PK, different values)."""
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,31
-        2,Bobby,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("modified_rows", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("modified_rows") }}', "primary_keys": ["customer_id"]}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    # Two rows modified (customer_id=1,2), each appears twice (old and new)
-    assert len(run_result.diff.data) == 4
-
-
-def test_query_diff_in_warehouse_composite_primary_key(dbt_test_helper):
-    """Test _query_diff_join with composite (multi-column) primary key."""
-    csv_data_base = """
-        region,customer_id,name,revenue
-        US,1,Alice,100
-        US,2,Bob,200
-        EU,1,Charlie,150
-        """
-
-    csv_data_curr = """
-        region,customer_id,name,revenue
-        US,1,Alice,100
-        US,2,Bob,250
-        EU,1,Charlie,150
-        """
-
-    dbt_test_helper.create_model("composite_pk", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("composite_pk") }}', "primary_keys": ["region", "customer_id"]}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    # One row modified (US,2), appears twice
-    assert len(run_result.diff.data) == 2
-
-    # Verify primary_keys preserved as list
-    assert isinstance(task.params.primary_keys, list)
-    assert len(task.params.primary_keys) == 2
-
-
-def test_query_diff_in_warehouse_empty_base(dbt_test_helper):
-    """Test _query_diff_join when base returns no rows."""
-    csv_data_base = """
-        customer_id,name,age
-        """
-
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        """
-
-    dbt_test_helper.create_model("empty_base_wh", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("empty_base_wh") }}', "primary_keys": ["customer_id"]}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    # All current rows are "added" (in_a=False, in_b=True)
-    assert len(run_result.diff.data) == 2
-    for row in run_result.diff.data:
-        in_a_idx = next(i for i, c in enumerate(run_result.diff.columns) if c.key.lower() == "in_a")
-        in_b_idx = next(i for i, c in enumerate(run_result.diff.columns) if c.key.lower() == "in_b")
-        assert row[in_a_idx] is False
-        assert row[in_b_idx] is True
-
-
-def test_query_diff_in_warehouse_empty_current(dbt_test_helper):
-    """Test _query_diff_join when current returns no rows."""
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        """
-
-    csv_data_curr = """
-        customer_id,name,age
-        """
-
-    dbt_test_helper.create_model("empty_curr_wh", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("empty_curr_wh") }}', "primary_keys": ["customer_id"]}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    # All base rows are "removed" (in_a=True, in_b=False)
-    assert len(run_result.diff.data) == 2
-    for row in run_result.diff.data:
-        in_a_idx = next(i for i, c in enumerate(run_result.diff.columns) if c.key.lower() == "in_a")
-        in_b_idx = next(i for i, c in enumerate(run_result.diff.columns) if c.key.lower() == "in_b")
-        assert row[in_a_idx] is True
-        assert row[in_b_idx] is False
-
-
-def test_query_diff_in_warehouse_with_nulls(dbt_test_helper):
-    """Test _query_diff_join handles NULL values correctly."""
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,30
-        2,,25
-        3,Charlie,
-        """
-
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("nulls_wh", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("nulls_wh") }}', "primary_keys": ["customer_id"]}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    # Rows 2 and 3 have changes (null to value), each appears twice
-    assert len(run_result.diff.data) == 4
-
-
-def test_query_diff_in_warehouse_columns_normalized(dbt_test_helper):
-    """Test that in_a/in_b column keys are normalized to lowercase."""
-    csv_data_base = """
-        customer_id,name
-        1,Alice
-        """
-
-    csv_data_curr = """
-        customer_id,name
-        1,Alice
-        2,Bob
-        """
-
-    dbt_test_helper.create_model("normalized_cols", csv_data_base, csv_data_curr)
-    params = {"sql_template": 'select * from {{ ref("normalized_cols") }}', "primary_keys": ["customer_id"]}
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-
-    # Check in_a/in_b columns are lowercase
-    column_keys = [c.key for c in run_result.diff.columns]
-    assert "in_a" in column_keys
-    assert "in_b" in column_keys
-    # Should not have uppercase variants
-    assert "IN_A" not in column_keys
-    assert "IN_B" not in column_keys
-
-
-def test_query_diff_primary_keys_normalized(dbt_test_helper):
-    """Test that primary_keys are normalized to match actual column keys."""
-    csv_data = """
-        CUSTOMER_ID,name,age
-        1,Alice,30
-        2,Bob,25
-        """
-
-    dbt_test_helper.create_model("pk_normalize", csv_data, csv_data)
-    # User provides lowercase, but column might be uppercase in warehouse
-    params = {"sql_template": 'select * from {{ ref("pk_normalize") }}', "primary_keys": ["customer_id"]}
-    task = QueryDiffTask(params)
-    task.execute()
-
-    # After normalization, primary_keys should match actual column key
-    # The exact casing depends on the warehouse (DuckDB preserves case from CSV header)
-    assert task.params.primary_keys is not None
-    assert len(task.params.primary_keys) == 1
-
-
-# =============================================================================
-# Validator Tests
-# =============================================================================
-
-
-def test_validator():
-    from recce.tasks.query import QueryCheckValidator, QueryDiffCheckValidator
-
-    def validate(params=None, view_options=None):
-        if view_options is None:
-            view_options = {}
-        if params is None:
-            params = {}
-        QueryCheckValidator().validate(
-            {
-                "name": "test",
-                "type": "query",
-                "params": params,
-                "view_options": view_options,
-            }
-        )
-
-    def validate_diff(params=None, view_options=None):
-        if view_options is None:
-            view_options = {}
-        if params is None:
-            params = {}
-        QueryDiffCheckValidator().validate(
-            {
-                "name": "test",
-                "type": "query_diff",
-                "params": params,
-                "view_options": view_options,
-            }
-        )
-
-    # query
-    validate({"sql_template": "select * from abc"})
-
-    # diff in client
-    validate_diff({"sql_template": "select * from abc"})
-    validate_diff(
-        {
-            "sql_template": "select * from abc",
-            "base_sql_template": "select * from abc",
-        }
-    )
-
-    # diff in warehouse
-    validate_diff(
-        {
-            "primary_keys": ["customer_id"],
-            "sql_template": "select * from abc",
-        }
-    )
-    validate_diff(
-        {
-            "sql_template": "select * from abc",
-            "base_sql_template": "select * from abc",
-            "primary_keys": ["customer_id"],
-        }
-    )
-
-    # invalid
-    with pytest.raises(ValueError):
-        validate()
-
-    with pytest.raises(ValueError):
-        validate_diff()
-
-    with pytest.raises(ValueError):
-        validate_diff({"sql_template": 123, "primary_keys": "xyz"})
-
-    with pytest.raises(ValueError):
-        validate_diff({"sql_template": "s", "primary_keys": "xyz"})

package/recce-source/tests/tasks/test_row_count.py

@@ -1,133 +0,0 @@
-import pytest
-
-from recce.tasks import RowCountDiffTask
-
-
-def test_row_count(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        """
-
-    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr, unique_id="model.customers")
-    task = RowCountDiffTask(dict(node_names=["customers"]))
-    run_result = task.execute()
-    assert run_result["customers"]["base"] == 2
-    assert run_result["customers"]["curr"] == 3
-
-    task = RowCountDiffTask(dict(node_names=["customers_"]))
-    run_result = task.execute()
-    assert run_result["customers_"]["base"] is None
-    assert run_result["customers_"]["curr"] is None
-
-    task = RowCountDiffTask(dict(node_ids=["model.customers"]))
-    run_result = task.execute()
-    assert run_result["customers"]["base"] == 2
-    assert run_result["customers"]["curr"] == 3
-
-
-def test_row_count_with_selector(dbt_test_helper):
-    csv_data_1 = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_2 = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        """
-
-    dbt_test_helper.create_model("model_1", csv_data_1, csv_data_2, depends_on=[])
-    dbt_test_helper.create_model("model_2", csv_data_1, csv_data_1, depends_on=["model_1"])
-    task = RowCountDiffTask(dict(select="model_1"))
-    run_result = task.execute()
-    assert len(run_result) == 1
-
-    task = RowCountDiffTask(dict(select="model_1+"))
-    run_result = task.execute()
-    assert len(run_result) == 2
-
-
-def test_validator():
-    from recce.tasks.rowcount import RowCountDiffCheckValidator
-
-    validator = RowCountDiffCheckValidator()
-
-    def validate(params: dict):
-        validator.validate(
-            {
-                "name": "test",
-                "type": "row_count_diff",
-                "params": params,
-            }
-        )
-
-    # Select all modesl
-    validate({})
-
-    # Select by node name
-    validate(
-        {
-            "node_names": ["abc"],
-        }
-    )
-    with pytest.raises(ValueError):
-        validate(
-            {
-                "node_names": "abc",
-            }
-        )
-
-    # Select by node id
-    validate(
-        {
-            "node_ids": ["model.abc"],
-        }
-    )
-
-    # Select by selector
-    validate(
-        {
-            "select": "customers",
-            "exclude": "customers",
-            "packages": ["jaffle_shop"],
-            "view_mode": "all",
-        }
-    )
-
-    # packages should be an array
-    with pytest.raises(ValueError):
-        validate(
-            {
-                "packages": "jaffle_shop",
-            }
-        )
-
-    # view_mode should be 'all' or 'changed_models'
-    validate(
-        {
-            "view_mode": None,
-        }
-    )
-    validate(
-        {
-            "view_mode": "all",
-        }
-    )
-    with pytest.raises(ValueError):
-        validate(
-            {
-                "view_mode": "abc",
-            }
-        )