@datarecce/ui 0.1.30 → 0.1.32

package/recce-source/tests/state/test_local.py

@@ -0,0 +1,164 @@
+import os
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from recce.models import Check, Run, RunType
+from recce.state import FileStateLoader, RecceState, RecceStateLoader
+
+
+class TestFileStateLoader(unittest.TestCase):
+    def setUp(self):
+        self.temp_file = None
+        self.temp_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        if self.temp_file and os.path.exists(self.temp_file):
+            os.remove(self.temp_file)
+
+    def test_init_with_defaults(self):
+        loader = FileStateLoader()
+        self.assertFalse(loader.review_mode)
+        self.assertFalse(loader.cloud_mode)
+        self.assertIsNone(loader.state_file)
+        self.assertIsNone(loader.state)
+
+    def test_init_with_parameters(self):
+        state_file = "/path/to/state.json"
+        initial_state = RecceState()
+        loader = FileStateLoader(review_mode=True, state_file=state_file, initial_state=initial_state)
+        self.assertTrue(loader.review_mode)
+        self.assertFalse(loader.cloud_mode)
+        self.assertEqual(loader.state_file, state_file)
+        self.assertEqual(loader.state, initial_state)
+
+    def test_verify_review_mode_without_state_file(self):
+        loader = FileStateLoader(review_mode=True, state_file=None)
+        self.assertFalse(loader.verify())
+        self.assertEqual(loader.error_message, "Recce can not launch without a state file.")
+        self.assertEqual(loader.hint_message, "Please provide a state file in the command argument.")
+
+    def test_verify_review_mode_with_state_file(self):
+        loader = FileStateLoader(review_mode=True, state_file="/path/to/state.json")
+        self.assertTrue(loader.verify())
+        self.assertIsNone(loader.error_message)
+        self.assertIsNone(loader.hint_message)
+
+    def test_verify_non_review_mode(self):
+        loader = FileStateLoader(review_mode=False, state_file=None)
+        self.assertTrue(loader.verify())
+
+    def test_load_state_with_existing_file(self):
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
+            self.temp_file = f.name
+            run = Run(type=RunType.QUERY, params=dict(sql_template="select 1"))
+            check = Check(name="test check", description="test", type=run.type, params=run.params)
+            state = RecceState(runs=[run], checks=[check])
+            f.write(state.to_json())
+
+        loader = FileStateLoader(state_file=self.temp_file)
+        loaded_state, etag = loader._load_state()
+
+        self.assertIsInstance(loaded_state, RecceState)
+        self.assertEqual(len(loaded_state.runs), 1)
+        self.assertEqual(len(loaded_state.checks), 1)
+        self.assertIsNone(etag)
+
+    def test_load_state_without_file(self):
+        loader = FileStateLoader(state_file=None)
+        state, etag = loader._load_state()
+
+        self.assertIsNone(state)
+        self.assertIsNone(etag)
+
+    def test_export_state_without_file(self):
+        loader = FileStateLoader(state_file=None)
+        loader.state = RecceState()
+
+        message, tag = loader._export_state()
+
+        self.assertEqual(message, "No state file is provided. Skip storing the state.")
+        self.assertIsNone(tag)
+
+    def test_export_state_with_file(self):
+        with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f:
+            self.temp_file = f.name
+
+        run = Run(type=RunType.QUERY, params=dict(sql_template="select 1"))
+        initial_state = RecceState(runs=[run])
+
+        loader = FileStateLoader(state_file=self.temp_file, initial_state=initial_state)
+
+        with patch("recce.state.local.logger") as mock_logger:
+            message, tag = loader._export_state()
+
+        self.assertTrue(message.startswith("The state file is stored at"))
+        self.assertIn(self.temp_file, message)
+        self.assertIsNone(tag)
+        mock_logger.info.assert_called_once()
+
+        # Verify file was actually written
+        self.assertTrue(os.path.exists(self.temp_file))
+        loaded_state = RecceState.from_file(self.temp_file)
+        self.assertEqual(len(loaded_state.runs), 1)
+
+    def test_purge_with_existing_file(self):
+        with tempfile.NamedTemporaryFile(delete=False) as f:
+            self.temp_file = f.name
+            # Write valid JSON content for initial state
+            f.write(b'{"runs": [], "checks": []}')
+
+        loader = FileStateLoader(state_file=self.temp_file)
+        result = loader.purge()
+
+        self.assertTrue(result)
+        self.assertFalse(os.path.exists(self.temp_file))
+        self.assertIsNone(loader.error_message)
+
+    def test_purge_with_nonexistent_file(self):
+        nonexistent_file = "/nonexistent/path/file.json"
+        loader = FileStateLoader(state_file=nonexistent_file)
+
+        result = loader.purge()
+
+        self.assertFalse(result)
+        self.assertIsNotNone(loader.error_message)
+        self.assertTrue(loader.error_message.startswith("Failed to remove the state file:"))
+
+    def test_purge_without_file(self):
+        loader = FileStateLoader(state_file=None)
+
+        result = loader.purge()
+
+        self.assertFalse(result)
+        self.assertEqual(loader.error_message, "No state file is provided. Skip removing the state file.")
+
+    def test_inheritance_from_state_loader(self):
+        loader = FileStateLoader()
+        self.assertIsInstance(loader, RecceStateLoader)
+
+    def test_load_method_calls_load_state(self):
+        loader = FileStateLoader(state_file=None)
+
+        with patch.object(loader, "_load_state") as mock_load_state:
+            mock_load_state.return_value = (RecceState(), None)
+            result = loader.load()
+
+            mock_load_state.assert_called_once()
+            self.assertIsInstance(result, RecceState)
+
+    def test_export_method_calls_export_state(self):
+        loader = FileStateLoader(state_file=None)
+        loader.state = RecceState()
+
+        with patch.object(loader, "_export_state") as mock_export_state:
+            mock_export_state.return_value = ("message", None)
+            with patch("recce.state.local.logger"):
+                result = loader.export()
+
+            mock_export_state.assert_called_once()
+            self.assertEqual(result, "message")
+
+
+if __name__ == "__main__":
+    unittest.main()

package/recce-source/tests/state/test_state_loader.py

@@ -0,0 +1,211 @@
+import unittest
+from unittest.mock import Mock, patch
+
+from recce.exceptions import RecceException
+from recce.state import RecceState, RecceStateLoader
+from recce.state.const import RECCE_API_TOKEN_MISSING
+
+
+class ConcreteStateLoader(RecceStateLoader):
+    """Concrete implementation for testing the abstract base class"""
+
+    def __init__(self, **kwargs):
+        self.verify_result = True
+        self.load_state_result = (RecceState(), "test_etag")
+        self.export_state_result = ("exported", "test_etag")
+        self.purge_result = True
+        super().__init__(**kwargs)
+
+    def verify(self) -> bool:
+        return self.verify_result
+
+    def _load_state(self):
+        return self.load_state_result
+
+    def _export_state(self):
+        return self.export_state_result
+
+    def purge(self) -> bool:
+        return self.purge_result
+
+
+class TestRecceStateLoader(unittest.TestCase):
+
+    @patch("recce.state.state_loader.fetch_pr_metadata")
+    def test_init_with_parameters(self, mock_fetch_pr):
+        mock_pr_info = Mock()
+        mock_pr_info.id = "123"
+        mock_fetch_pr.return_value = mock_pr_info
+
+        initial_state = RecceState()
+        cloud_options = {"github_token": "test_token"}
+
+        loader = ConcreteStateLoader(
+            review_mode=True,
+            cloud_mode=True,
+            state_file="/path/to/file.json",
+            cloud_options=cloud_options,
+            initial_state=initial_state,
+        )
+
+        self.assertTrue(loader.review_mode)
+        self.assertTrue(loader.cloud_mode)
+        self.assertEqual(loader.state_file, "/path/to/file.json")
+        self.assertEqual(loader.cloud_options, cloud_options)
+        self.assertEqual(loader.state, initial_state)
+
+    @patch("recce.state.state_loader.fetch_pr_metadata")
+    def test_cloud_mode_with_github_token(self, mock_fetch_pr):
+        mock_pr_info = Mock()
+        mock_pr_info.id = "123"
+        mock_fetch_pr.return_value = mock_pr_info
+
+        cloud_options = {"github_token": "test_token"}
+        loader = ConcreteStateLoader(cloud_mode=True, cloud_options=cloud_options)
+
+        self.assertEqual(loader.catalog, "github")
+        self.assertEqual(loader.pr_info, mock_pr_info)
+        mock_fetch_pr.assert_called_once_with(cloud=True, github_token="test_token")
+
+    @patch("recce.state.state_loader.fetch_pr_metadata")
+    def test_cloud_mode_with_github_token_no_pr_id(self, mock_fetch_pr):
+        mock_pr_info = Mock()
+        mock_pr_info.id = None
+        mock_fetch_pr.return_value = mock_pr_info
+
+        cloud_options = {"github_token": "test_token"}
+
+        with self.assertRaises(RecceException) as cm:
+            ConcreteStateLoader(cloud_mode=True, cloud_options=cloud_options)
+
+        self.assertEqual(str(cm.exception), "Cannot get the pull request information from GitHub.")
+
+    def test_cloud_mode_with_api_token(self):
+        cloud_options = {"api_token": "test_api_token", "share_id": "test_share_id"}
+        loader = ConcreteStateLoader(cloud_mode=True, cloud_options=cloud_options)
+
+        self.assertEqual(loader.catalog, "preview")
+        self.assertEqual(loader.share_id, "test_share_id")
+
+    def test_cloud_mode_without_tokens(self):
+        with self.assertRaises(RecceException) as cm:
+            ConcreteStateLoader(cloud_mode=True, cloud_options={})
+
+        self.assertEqual(str(cm.exception), RECCE_API_TOKEN_MISSING.error_message)
+
+    def test_error_and_hint_property(self):
+        loader = ConcreteStateLoader()
+        loader.error_message = "test error"
+        loader.hint_message = "test hint"
+
+        error, hint = loader.error_and_hint
+        self.assertEqual(error, "test error")
+        self.assertEqual(hint, "test hint")
+
+    def test_load_with_existing_state(self):
+        initial_state = RecceState()
+        loader = ConcreteStateLoader(initial_state=initial_state)
+
+        # Should return existing state without calling _load_state
+        result = loader.load(refresh=False)
+        self.assertEqual(result, initial_state)
+
+    def test_load_with_refresh(self):
+        initial_state = RecceState()
+        loader = ConcreteStateLoader(initial_state=initial_state)
+        new_state = RecceState()
+        loader.load_state_result = (new_state, "new_etag")
+
+        result = loader.load(refresh=True)
+        self.assertEqual(result, new_state)
+        self.assertEqual(loader.state_etag, "new_etag")
+
+    def test_export_updates_state(self):
+        loader = ConcreteStateLoader()
+        new_state = RecceState()
+
+        with patch("recce.state.state_loader.logger"):
+            result = loader.export(new_state)
+
+        self.assertEqual(loader.state, new_state)
+        self.assertEqual(result, "exported")
+
+    def test_save_as_in_cloud_mode(self):
+        cloud_options = {"api_token": "test_token", "share_id": "test_share"}
+        loader = ConcreteStateLoader(cloud_mode=True, cloud_options=cloud_options)
+
+        with self.assertRaises(Exception) as cm:
+            loader.save_as("/path/to/file.json")
+
+        self.assertEqual(str(cm.exception), "Cannot save the state to Recce Cloud.")
+
+    def test_save_as_in_local_mode(self):
+        loader = ConcreteStateLoader(cloud_mode=False)
+        state = RecceState()
+
+        with patch.object(loader, "export") as mock_export:
+            loader.save_as("/path/to/file.json", state)
+
+        self.assertEqual(loader.state_file, "/path/to/file.json")
+        mock_export.assert_called_once_with(state)
+
+    def test_info_without_state(self):
+        loader = ConcreteStateLoader()
+        loader.state = None
+
+        result = loader.info()
+
+        self.assertIsNone(result)
+        self.assertEqual(loader.error_message, "No state is loaded.")
+
+    def test_info_local_mode(self):
+        loader = ConcreteStateLoader(cloud_mode=False, state_file="/path/to/state.json")
+        loader.state = RecceState()
+
+        result = loader.info()
+
+        expected = {
+            "mode": "local",
+            "source": "/path/to/state.json",
+        }
+        self.assertEqual(result, expected)
+
+    @patch("recce.state.state_loader.fetch_pr_metadata")
+    def test_info_cloud_mode_with_pr_info(self, mock_fetch_pr):
+        mock_pr_info = Mock()
+        mock_pr_info.id = "123"
+        mock_fetch_pr.return_value = mock_pr_info
+
+        cloud_options = {"github_token": "test_token"}
+        loader = ConcreteStateLoader(cloud_mode=True, cloud_options=cloud_options)
+        loader.state = RecceState()
+
+        result = loader.info()
+
+        expected = {
+            "mode": "cloud",
+            "source": "Recce Cloud",
+            "pull_request": mock_pr_info,
+        }
+        self.assertEqual(result, expected)
+
+    def test_update_state(self):
+        loader = ConcreteStateLoader()
+        new_state = RecceState()
+
+        loader.update(new_state)
+        self.assertEqual(loader.state, new_state)
+
+    def test_refresh(self):
+        loader = ConcreteStateLoader()
+        new_state = RecceState()
+        loader.load_state_result = (new_state, "new_etag")
+
+        result = loader.refresh()
+
+        self.assertEqual(result, new_state)
+        self.assertEqual(loader.state, new_state)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/recce-source/tests/tasks/conftest.py

@@ -0,0 +1,4 @@
+# Share the fixtures for the tasks tests
+
+# noinspection PyUnresolvedReferences
+from tests.adapter.dbt_adapter.conftest import dbt_test_helper  # noqa: F401

package/recce-source/tests/tasks/test_histogram.py

@@ -0,0 +1,129 @@
+import pytest
+
+from recce.tasks.histogram import (
+    HistogramDiffCheckValidator,
+    HistogramDiffTask,
+    _is_histogram_supported,
+)
+
+
+def test_histogram(dbt_test_helper):
+    csv_data = """
+        customer_id,name,age
+        1,Alice,30
+        2,Bob,25
+        3,Charlie,35
+        4,Dolly,50
+        """
+
+    dbt_test_helper.create_model("customers", csv_data, csv_data)
+
+    params = {"model": "customers", "column_name": "age", "column_type": "int"}
+
+    task = HistogramDiffTask(params)
+    run_result = task.execute()
+
+    # {
+    #     'base': {'counts': [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], 'total': 4},
+    #     'current': {'counts': [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], 'total': 4},
+    #     'min': 25, 'max': 50,
+    #     'bin_edges': [25, 26, ..., 51],
+    #     'labels': ['25-26', ..., '51-52']
+    # }
+    assert run_result["current"]["counts"][0] == 1
+    assert run_result["current"]["counts"][-1] == 1
+    assert run_result["current"]["total"] == 4
+    assert run_result["min"] == 25
+    assert run_result["max"] == 50
+    assert run_result["bin_edges"][0] == 25
+    assert run_result["bin_edges"][-1] == 51
+
+
+def test_histogram_emtpy(dbt_test_helper):
+    csv_data = """
+    customer_id,name,age
+    1,Alice,30
+    2,Bob,25
+    3,Charlie,35
+    4,Dolly,50
+    """
+
+    csv_data_zero = """
+    customer_id,name,age
+    """
+
+    dbt_test_helper.create_model("customers", csv_data_zero, csv_data_zero)
+    dbt_test_helper.create_model("customers2", csv_data, csv_data_zero)
+    dbt_test_helper.create_model("customers3", csv_data_zero, csv_data)
+
+    params = {"model": "customers", "column_name": "age", "column_type": "int"}
+
+    task = HistogramDiffTask(params)
+    run_result = task.execute()
+
+    assert len(run_result["current"]["counts"]) == 0
+    assert run_result["current"]["total"] == 0
+    assert run_result["min"] is None
+    assert run_result["max"] is None
+    assert len(run_result["bin_edges"]) == 0
+
+    params = {"model": "customers2", "column_name": "age", "column_type": "int"}
+
+    task = HistogramDiffTask(params)
+    run_result = task.execute()
+    assert run_result["base"]["counts"][0] == 1
+    assert run_result["base"]["counts"][-1] == 1
+    assert run_result["base"]["total"] == 4
+    assert run_result["current"]["counts"][0] == 0
+    assert run_result["current"]["counts"][-1] == 0
+    assert run_result["current"]["total"] == 0
+    assert run_result["min"] == 25
+    assert run_result["max"] == 50
+    assert run_result["bin_edges"][0] == 25
+    assert run_result["bin_edges"][-1] == 51
+
+    params = {"model": "customers3", "column_name": "age", "column_type": "int"}
+
+    task = HistogramDiffTask(params)
+    run_result = task.execute()
+    assert run_result["base"]["counts"][0] == 0
+    assert run_result["base"]["counts"][-1] == 0
+    assert run_result["base"]["total"] == 0
+    assert run_result["current"]["counts"][0] == 1
+    assert run_result["current"]["counts"][-1] == 1
+    assert run_result["current"]["total"] == 4
+    assert run_result["min"] == 25
+    assert run_result["max"] == 50
+    assert run_result["bin_edges"][0] == 25
+    assert run_result["bin_edges"][-1] == 51
+
+
+def test_validator():
+    def validate(params: dict = {}, view_options: dict = {}):
+        HistogramDiffCheckValidator().validate(
+            {
+                "name": "test",
+                "type": "histogram_diff",
+                "params": params,
+                "view_options": view_options,
+            }
+        )
+
+    validate(
+        {
+            "model": "customers",
+            "column_name": "age",
+            "column_type": "int",
+        }
+    )
+
+    with pytest.raises(ValueError):
+        validate({})
+
+
+def test_is_column_type_supported_by_histogram():
+    assert _is_histogram_supported("varchar") is False
+    assert _is_histogram_supported("varchar(16)") is False
+    assert _is_histogram_supported("varchar(256)") is False
+    assert _is_histogram_supported("bool") is False
+    assert _is_histogram_supported("int") is True

package/recce-source/tests/tasks/test_lineage.py

@@ -0,0 +1,55 @@
+import pytest
+
+
+def test_validator():
+    from recce.tasks.lineage import LineageDiffCheckValidator
+
+    validator = LineageDiffCheckValidator()
+
+    def validate(params: dict):
+        validator.validate(
+            {
+                "name": "test",
+                "type": "schema_diff",
+                "params": params,
+            }
+        )
+
+    # Select all models
+    validate({})
+
+    # Select by selector
+    validate(
+        {
+            "select": "customers",
+            "exclude": "customers",
+            "packages": ["jaffle_shop"],
+            "view_mode": "all",
+        }
+    )
+
+    # packages should be an array
+    with pytest.raises(ValueError):
+        validate(
+            {
+                "packages": "jaffle_shop",
+            }
+        )
+
+    # view_mode should be 'all' or 'changed_models'
+    validate(
+        {
+            "view_mode": None,
+        }
+    )
+    validate(
+        {
+            "view_mode": "all",
+        }
+    )
+    with pytest.raises(ValueError):
+        validate(
+            {
+                "view_mode": "abc",
+            }
+        )

package/recce-source/tests/tasks/test_preset_checks.py

@@ -0,0 +1,64 @@
+import pytest
+
+
+def test_default_validator():
+    from recce.tasks.core import CheckValidator
+
+    CheckValidator().validate(
+        {
+            "name": "test",
+            "type": "row_count_diff",
+            "params": {},
+        }
+    )
+
+    with pytest.raises(ValueError):
+        CheckValidator().validate(
+            {
+                "type": "row_count_diff",
+            }
+        )
+
+
+def test_query_diff_validator():
+    from recce.tasks.query import QueryDiffCheckValidator
+
+    QueryDiffCheckValidator().validate(
+        {
+            "name": "test",
+            "type": "query_diff",
+            "params": {"sql_template": "select * from {{ model }}"},
+        }
+    )
+    with pytest.raises(ValueError):
+        QueryDiffCheckValidator().validate(
+            {
+                "name": "test",
+                "type": "query_diff",
+            }
+        )
+
+
+def test_lineage_diff_validator():
+    from recce.tasks.lineage import LineageDiffCheckValidator
+
+    LineageDiffCheckValidator().validate(
+        {
+            "name": "test",
+            "type": "lineage_diff",
+            "params": {},
+        }
+    )
+    LineageDiffCheckValidator().validate(
+        {
+            "name": "test",
+            "type": "lineage_diff",
+            "view_options": {},
+        }
+    )
+    with pytest.raises(ValueError):
+        LineageDiffCheckValidator().validate(
+            {
+                "type": "lineage_diff",
+            }
+        )