@datarecce/ui 0.1.30 → 0.1.31

package/recce-source/tests/adapter/dbt_adapter/test_proj/package-lock.yml

@@ -0,0 +1,8 @@
+packages:
+- package: dbt-labs/dbt_utils
+  version: 1.1.1
+- package: data-mie/dbt_profiler
+  version: 0.8.1
+- package: dbt-labs/audit_helper
+  version: 0.11.0
+sha1_hash: 3759eaf120cdd1d41e53b6d3ae99e2361147350a

package/recce-source/tests/adapter/dbt_adapter/test_proj/packages.yml

@@ -0,0 +1,7 @@
+packages:
+  - package: dbt-labs/dbt_utils
+    version: ">=1.1.1"
+  - package: data-mie/dbt_profiler
+    version: ">=0.8.1"
+  - package: dbt-labs/audit_helper
+    version: ">=0.11.0"

package/recce-source/tests/adapter/dbt_adapter/test_proj/profiles.yml

@@ -0,0 +1,6 @@
+recce_test:
+  target: dev
+  outputs:
+    dev:
+      type: duckdb
+      # path: 'recce_test.duckdb'

package/recce-source/tests/adapter/dbt_adapter/test_selector.py

@@ -0,0 +1,205 @@
+from recce.adapter.dbt_adapter import DbtAdapter
+
+
+def test_select(dbt_test_helper):
+    csv_data_curr = """
+        customer_id,name,age
+        1,Alice,30
+        2,Bob,25
+        3,Charlie,35
+        """
+
+    csv_data_base = """
+        customer_id,name,age
+        1,Alice,35
+        2,Bob,25
+        3,Charlie,35
+        """
+
+    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_curr)
+    dbt_test_helper.create_model("customers_2", csv_data_base, csv_data_base, depends_on=["customers_1"])
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+
+    # Test methods
+    node_ids = adapter.select_nodes("customers_1")
+    assert len(node_ids) == 1
+    node_ids = adapter.select_nodes("resource_type:model")
+    assert len(node_ids) == 2
+    node_ids = adapter.select_nodes("tag:test_tag")
+    assert len(node_ids) == 2
+    node_ids = adapter.select_nodes("tag:test_tag2")
+    assert len(node_ids) == 0
+    node_ids = adapter.select_nodes("config.materialized:incremental")
+    assert len(node_ids) == 0
+    node_ids = adapter.select_nodes("config.materialized:table")
+    assert len(node_ids) == 2
+
+    # Test state method
+    node_ids = adapter.select_nodes("state:modified")
+    assert len(node_ids) == 1
+
+    # Test set operation
+    node_ids = adapter.select_nodes("customers_1 customers_2")
+    assert len(node_ids) == 2
+    node_ids = adapter.select_nodes("customers_1,customers_2")
+    assert len(node_ids) == 0
+    node_ids = adapter.select_nodes("config.materialized:table,tag:test_tag")
+    assert len(node_ids) == 2
+    node_ids = adapter.select_nodes("config.materialized:table,tag:test_tag2")
+    assert len(node_ids) == 0
+    node_ids = adapter.select_nodes(exclude="customers_1")
+    assert len(node_ids) == 1
+    node_ids = adapter.select_nodes("customers_1", exclude="customers_2")
+    assert len(node_ids) == 1
+
+    # Test graph operation
+    node_ids = adapter.select_nodes("state:modified+")
+    assert len(node_ids) == 2
+    node_ids = adapter.select_nodes("+state:modified")
+    assert len(node_ids) == 1
+
+    # Test resource type: snapshot
+    dbt_test_helper.create_snapshot("snapshot_1", csv_data_base, csv_data_curr)
+    dbt_test_helper.create_model("use_snapshot", csv_data_base, csv_data_base, depends_on=["snapshot_1"])
+
+    node_ids = adapter.select_nodes("resource_type:snapshot")
+    assert len(node_ids) == 1
+
+    node_ids = adapter.select_nodes("resource_type:snapshot+")
+    assert len(node_ids) == 2
+
+    node_ids = adapter.select_nodes("state:modified,resource_type:snapshot")
+    assert len(node_ids) == 1
+
+
+def test_select_removed_by_graph(dbt_test_helper):
+    csv_data_curr = """
+        customer_id,name,age
+        1,Alice,30
+        2,Bob,25
+        3,Charlie,35
+        """
+
+    csv_data_base = """
+        customer_id,name,age
+        1,Alice,35
+        2,Bob,25
+        3,Charlie,35
+        """
+
+    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_curr)
+    dbt_test_helper.create_model("customers_2", base_csv=csv_data_base, depends_on=["customers_1"])
+    dbt_test_helper.create_model("customers_3", curr_csv=csv_data_curr, depends_on=["customers_1"])
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+
+    # Test graph operation
+    node_ids = adapter.select_nodes("customers_1+")
+    assert len(node_ids) == 3
+    node_ids = adapter.select_nodes("+customers_2")
+    assert len(node_ids) == 2
+    node_ids = adapter.select_nodes("+customers_3")
+    assert len(node_ids) == 2
+
+
+def test_select_source_removed_by_graph(dbt_test_helper):
+    csv_data_curr = """
+        customer_id,name,age
+        1,Alice,30
+        2,Bob,25
+        3,Charlie,35
+        """
+
+    csv_data_base = """
+        customer_id,name,age
+        1,Alice,35
+        2,Bob,25
+        3,Charlie,35
+        """
+
+    dbt_test_helper.create_model("customers_2", base_csv=csv_data_base)
+    dbt_test_helper.create_model("customers_3", curr_csv=csv_data_curr)
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+
+    # Test graph operation
+    node_ids = adapter.select_nodes("state:modified")
+    assert len(node_ids) == 2
+
+
+def test_select_with_disabled(dbt_test_helper):
+    csv_data_curr = """
+        customer_id,name,age
+        1,Alice,30
+        2,Bob,25
+        3,Charlie,35
+        """
+
+    csv_data_base = """
+        customer_id,name,age
+        1,Alice,35
+        2,Bob,25
+        3,Charlie,35
+        """
+
+    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_curr)
+    dbt_test_helper.create_model("customers_2", csv_data_base, csv_data_curr, disabled=True)
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+
+    # Test graph operation
+    node_ids = adapter.select_nodes("customers_1+")
+    assert len(node_ids) == 1
+
+
+def test_select_with_pacakage_mode_include_exclude(dbt_test_helper):
+    csv_data_curr = """
+            customer_id,name,age
+            1,Alice,30
+            2,Bob,25
+            3,Charlie,35
+            """
+
+    csv_data_base = """
+            customer_id,name,age
+            1,Alice,35
+            2,Bob,25
+            3,Charlie,35
+            """
+
+    """
+    The diagram of the models:
+    customers_1 ── customers_2 ── customers_3(*) ├── other_package.customers_4
+                                                 └── customers_5
+
+    Only customers_3 is changed
+    """
+    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_base)
+    dbt_test_helper.create_model("customers_2", csv_data_base, csv_data_base, depends_on=["customers_1"])
+    dbt_test_helper.create_model("customers_3", csv_data_base, csv_data_curr, depends_on=["customers_2"])
+    dbt_test_helper.create_model(
+        "customers_4", csv_data_base, csv_data_base, depends_on=["customers_3"], package_name="other_package"
+    )
+    dbt_test_helper.create_model("customers_5", csv_data_base, csv_data_base, depends_on=["customers_3"])
+
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+
+    node_ids = adapter.select_nodes(packages=["other_package"])
+    assert len(node_ids) == 1
+
+    node_ids = adapter.select_nodes()
+    assert len(node_ids) == 4
+
+    node_ids = adapter.select_nodes(view_mode="changed_models")
+    assert len(node_ids) == 3
+
+    node_ids = adapter.select_nodes(view_mode="changed_models", packages=["other_package"])
+    assert len(node_ids) == 1
+
+    node_ids = adapter.select_nodes(view_mode="changed_models", packages=["other_package"], exclude="customers_4")
+    assert len(node_ids) == 0
+    node_ids = adapter.select_nodes(view_mode="changed_models", packages=["other_package"], select="customers_1+")
+    assert len(node_ids) == 1
+    node_ids = adapter.select_nodes(view_mode="changed_models", select="+customers_5")
+    assert len(node_ids) == 3
+    node_ids = adapter.select_nodes(view_mode="all", select="+customers_5")
+    assert len(node_ids) == 4
+    node_ids = adapter.select_nodes(select="+customers_5")
+    assert len(node_ids) == 4

package/recce-source/tests/apis/row_count_diff.json

@@ -0,0 +1,59 @@
+{
+  "metadata": null,
+  "runs": [
+    {
+      "type": "row_count_diff",
+      "params": {
+        "node_names": [
+          "customers",
+          "customer_segments"
+        ]
+      },
+      "check_id": null,
+      "result": {
+        "customers": {
+          "base": 1856,
+          "curr": 1856
+        },
+        "customer_segments": {
+          "base": 1856,
+          "curr": 1856
+        }
+      },
+      "error": null,
+      "progress": null,
+      "run_id": "92f31d63-0758-46af-a674-0e969208ec96",
+      "run_at": "2024-03-11T07:20:31Z"
+    },
+    {
+      "type": "row_count_diff",
+      "params": {
+        "node_names": [
+          "customer_order_pattern"
+        ]
+      },
+      "check_id": null,
+      "result": {
+        "customer_order_pattern": {
+          "base": 1856,
+          "curr": 1856
+        }
+      },
+      "error": null,
+      "progress": null,
+      "run_id": "ecef5ebd-f820-453d-9daf-f4f2f1b06d2e",
+      "run_at": "2024-03-11T07:20:41Z"
+    }
+  ],
+  "checks": [
+    {
+      "name": "Check - 11 Mar 2024",
+      "description": "",
+      "type": "simple",
+      "params": null,
+      "view_options": null,
+      "check_id": "7187e630-f64f-4b14-b1c5-b2e157ab721d",
+      "is_checked": false
+    }
+  ]
+}