@datarecce/ui 0.1.41 → 0.2.0

package/recce-source/integration_tests/dbt/smoke_test.sh

@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-cd "$SCRIPT_DIR"
-pwd
-
-# Prepare env
-git restore models/customers.sql
-dbt --version
-dbt deps
-dbt seed --target-path target-base
-dbt run --target-path target-base
-dbt docs generate --target-path target-base
-
-echo "where customer_id > 0" >> models/customers.sql
-dbt run
-dbt docs generate
-git restore models/customers.sql
-
-# Recce Run
-mkdir -p ~/.recce
-echo "user_id: 00000000000000000000000000000000" > ~/.recce/profile.yml
-echo "anonymous_tracking: false" >> ~/.recce/profile.yml
-
-assert_string_value() {
-    if [ "$1" != "$2" ]; then
-        echo "Expected $2, but got $1"
-        exit 1
-    fi
-}
-
-recce run
-if ! [ -e recce_state.json ]; then
-    echo "recce_state.json not found"
-    exit 1
-fi
-
-model=$(cat recce_state.json | jq '.runs[0].result | keys | .[0]' | tr -d '"')
-run_type=$(cat recce_state.json | jq '.runs[0]'.type | tr -d '"')
-assert_string_value $model "customers"
-assert_string_value $run_type "row_count_diff"
-
-# Recce Summary
-recce summary ./recce_state.json | tee recce_summary.md
-cat ./recce_summary.md | grep -q customers
-
-# Recce Server
-function check_server_status() {
-    echo "Waiting for the server to respond..."
-    if timeout 20 bash -c 'until curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/api/info | grep -q 200; do
-    echo "Server not ready yet..."
-    sleep 2
-    done'; then
-        echo "Server is up and running."
-    else
-        echo "Failed to start the server within the time limit."
-        exit 1
-    fi
-
-    echo "Stopping the server..."
-    kill $(jobs -p) || true
-    echo "Server stopped."
-}
-
-echo "Starting the server..."
-recce server &
-check_server_status
-
-echo "Starting the server (review mode)..."
-recce server --review recce_state.json &
-check_server_status

package/recce-source/integration_tests/dbt/smoke_test_cloud.sh

@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-NEW_WORKSPACE=$(dirname "$GITHUB_WORKSPACE")
-cd "$NEW_WORKSPACE" || exit
-pwd
-
-# Clone jaffle_shop_duckdb
-GIT_REPO="https://github.com/DataRecce/jaffle_shop_duckdb.git"
-GIT_BRANCH="chore/smoke-test-cloud"
-
-git clone --depth 1 --branch $GIT_BRANCH $GIT_REPO
-cd jaffle_shop_duckdb || exit
-
-# Prepare dbt artifacts
-dbt --version
-dbt deps
-dbt seed
-dbt run
-dbt docs generate
-
-mkdir -p ~/.recce
-echo "user_id: 00000000000000000000000000000000" > ~/.recce/profile.yml
-echo "anonymous_tracking: false" >> ~/.recce/profile.yml
-
-# Hide PR information from GitHub Action
-HOLD_GITHUB_EVENT_PATH="$GITHUB_EVENT_PATH"
-unset GITHUB_EVENT_PATH
-
-# Recce artifacts
-recce cloud upload-artifacts --branch $GIT_BRANCH
-rm -rf target
-
-recce cloud download-base-artifacts --branch $GIT_BRANCH
-recce cloud download-artifacts --branch $GIT_BRANCH
-
-# Recce Run
-recce run --cloud
-
-# Recce state
-recce cloud download
-recce cloud purge --force
-recce cloud upload recce_state.json
-
-# Recce Summary
-recce summary --cloud
-
-function check_server_status() {
-    echo "Waiting for the server to respond..."
-    if timeout 20 bash -c 'until curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/api/info | grep -q 200; do
-    echo "Server not ready yet..."
-    sleep 2
-    done'; then
-        echo "Server is up and running."
-    else
-        echo "Failed to start the server within the time limit."
-        exit 1
-    fi
-
-    echo "Stopping the server..."
-    kill $(jobs -p) || true
-    echo "Server stopped."
-}
-
-# Recce Server
-echo "Starting the server (cloud and review mode)..."
-recce server --cloud --review &
-check_server_status
-recce cloud purge --force
-
-export GITHUB_EVENT_PATH="$HOLD_GITHUB_EVENT_PATH"

package/recce-source/integration_tests/sqlmesh/audits/assert_item_price_above_zero.sql

@@ -1,9 +0,0 @@
-AUDIT (
-    name assert_item_price_above_zero,
-    description "Asserts that the price of an item is above zero.",
-    standalone true
-);
-
-SELECT *
-FROM sushi.items
-WHERE price <= 0

package/recce-source/integration_tests/sqlmesh/audits/items.sql

@@ -1,7 +0,0 @@
-AUDIT (
-  name assert_items_price_exceeds_threshold
-);
-SELECT *
-FROM @this_model
-WHERE price <= @price;
-

package/recce-source/integration_tests/sqlmesh/audits/order_items.sql

@@ -1,7 +0,0 @@
-AUDIT (
-  name assert_order_items_quantity_exceeds_threshold
-);
-
-SELECT *
-FROM @this_model
-WHERE quantity <= @quantity

package/recce-source/integration_tests/sqlmesh/config.py

@@ -1,171 +0,0 @@
-import os
-
-from sqlmesh.core.config import (
-    AirflowSchedulerConfig,
-    AutoCategorizationMode,
-    BigQueryConnectionConfig,
-    CategorizerConfig,
-    Config,
-    DuckDBConnectionConfig,
-    EnvironmentSuffixTarget,
-    GatewayConfig,
-    ModelDefaultsConfig,
-    PlanConfig,
-    PostgresConnectionConfig,
-    SparkConnectionConfig,
-)
-from sqlmesh.core.notification_target import (
-    BasicSMTPNotificationTarget,
-    SlackApiNotificationTarget,
-    SlackWebhookNotificationTarget,
-)
-from sqlmesh.core.user import User, UserRole
-
-CURRENT_FILE_PATH = os.path.abspath(__file__)
-DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
-
-
-# An in memory DuckDB config.
-config = Config(
-    default_connection=DuckDBConnectionConfig(),
-    model_defaults=ModelDefaultsConfig(dialect="duckdb"),
-)
-
-bigquery_config = Config(
-    gateways={
-        "gcp": GatewayConfig(
-            connection=BigQueryConnectionConfig(),
-            state_connection=DuckDBConnectionConfig(database=f"{DATA_DIR}/bigquery.duckdb"),
-        )
-    },
-    default_gateway="gcp",
-    model_defaults=ModelDefaultsConfig(dialect="bigquery"),
-)
-
-# A configuration used for SQLMesh tests.
-test_config = Config(
-    gateways={"in_memory": GatewayConfig(connection=DuckDBConnectionConfig())},
-    default_gateway="in_memory",
-    plan=PlanConfig(auto_categorize_changes=CategorizerConfig(sql=AutoCategorizationMode.SEMI)),
-    model_defaults=ModelDefaultsConfig(dialect="duckdb"),
-)
-
-# A stateful DuckDB config.
-local_config = Config(
-    default_connection=DuckDBConnectionConfig(database=f"{DATA_DIR}/local.duckdb"),
-    model_defaults=ModelDefaultsConfig(dialect="duckdb"),
-)
-
-airflow_config = Config(
-    default_scheduler=AirflowSchedulerConfig(),
-    gateways=GatewayConfig(
-        connection=SparkConnectionConfig(
-            config_dir=os.path.join(CURRENT_FILE_PATH, "..", "airflow", "spark_conf"),
-            config={
-                "spark.hadoop.javax.jdo.option.ConnectionURL": "jdbc:postgresql://localhost:5432/metastore_db"
-            },
-        )
-    ),
-    model_defaults=ModelDefaultsConfig(dialect="duckdb", storage_format="iceberg"),
-)
-
-
-airflow_config_docker = Config(
-    default_scheduler=AirflowSchedulerConfig(airflow_url="http://airflow-webserver:8080/"),
-    gateways=GatewayConfig(connection=SparkConnectionConfig()),
-    model_defaults=ModelDefaultsConfig(dialect="duckdb", storage_format="iceberg"),
-)
-
-# A DuckDB config with a physical schema map.
-map_config = Config(
-    default_connection=DuckDBConnectionConfig(),
-    physical_schema_override={"sushi": "company_internal"},
-    model_defaults=ModelDefaultsConfig(dialect="duckdb"),
-)
-
-
-required_approvers_config = Config(
-    default_connection=DuckDBConnectionConfig(),
-    users=[
-        User(
-            username="admin",
-            roles=[UserRole.REQUIRED_APPROVER],
-            notification_targets=[
-                SlackApiNotificationTarget(
-                    notify_on=["apply_start", "apply_failure", "apply_end", "audit_failure"],
-                    token=os.getenv("ADMIN_SLACK_API_TOKEN"),
-                    channel="UXXXXXXXXX",  # User's Slack member ID
-                ),
-            ],
-        )
-    ],
-    notification_targets=[
-        SlackWebhookNotificationTarget(
-            notify_on=["apply_start", "apply_failure", "run_start"],
-            url=os.getenv("SLACK_WEBHOOK_URL"),
-        ),
-        BasicSMTPNotificationTarget(
-            notify_on=["run_failure"],
-            host=os.getenv("SMTP_HOST"),
-            user=os.getenv("SMTP_USER"),
-            password=os.getenv("SMTP_PASSWORD"),
-            sender="sushi@example.com",
-            recipients=[
-                "team@example.com",
-            ],
-        ),
-    ],
-    model_defaults=ModelDefaultsConfig(dialect="duckdb"),
-)
-
-
-environment_suffix_config = Config(
-    default_connection=DuckDBConnectionConfig(),
-    model_defaults=ModelDefaultsConfig(dialect="duckdb"),
-    environment_suffix_target=EnvironmentSuffixTarget.TABLE,
-)
-
-
-CATALOGS = {
-    "in_memory": ":memory:",
-    "other_catalog": ":memory:",
-}
-
-local_catalogs = Config(
-    default_connection=DuckDBConnectionConfig(catalogs=CATALOGS),
-    default_test_connection=DuckDBConnectionConfig(catalogs=CATALOGS),
-    model_defaults=ModelDefaultsConfig(dialect="duckdb"),
-)
-
-environment_catalog_mapping_config = Config(
-    default_connection=DuckDBConnectionConfig(
-        catalogs={
-            "physical": ":memory:",
-            "prod_catalog": ":memory:",
-            "dev_catalog": ":memory:",
-        }
-    ),
-    model_defaults=ModelDefaultsConfig(dialect="duckdb"),
-    environment_suffix_target=EnvironmentSuffixTarget.TABLE,
-    environment_catalog_mapping={
-        "^prod$": "prod_catalog",
-        ".*": "dev_catalog",
-    },
-)
-
-postgresql_config = Config(
-    # default_connection=DuckDBConnectionConfig(database=f"{DATA_DIR}/local.duckdb"),
-    default_connection=DuckDBConnectionConfig(database="gclouddata.duckdb"),
-    model_defaults=ModelDefaultsConfig(dialect="duckdb"),
-    gateways={
-        "my_gateway": GatewayConfig(
-            state_connection=PostgresConnectionConfig(
-                host="35.222.243.76",
-                port=5432,
-                user="unique-glider",
-                password="1d1dWu[NRYV5XUY3",
-                database="sushi-demo",
-            ),
-        ),
-    },
-)

package/recce-source/integration_tests/sqlmesh/helper.py

@@ -1,20 +0,0 @@
-import random
-import typing as t
-from datetime import datetime, timedelta
-
-import numpy as np
-
-
-def set_seed(dt: datetime) -> None:
-    """Helper function to set a seed so random choices are not so random."""
-    ts = int(dt.timestamp())
-    random.seed(ts)
-    np.random.seed(ts)
-
-
-def iter_dates(start: datetime, end: datetime) -> t.Generator[datetime, None, None]:
-    """Iterate through dates and set a deterministic seed."""
-    for i in range((end - start).days + 1):
-        dt = start + timedelta(days=i)
-        set_seed(dt)
-        yield dt

package/recce-source/integration_tests/sqlmesh/macros/macros.py

@@ -1,8 +0,0 @@
-from macros.utils import between  # type: ignore
-
-from sqlmesh import macro
-
-
-@macro()
-def incremental_by_ds(evaluator, column):
-    return between(evaluator, column, evaluator.locals["start_date"], evaluator.locals["end_date"])

package/recce-source/integration_tests/sqlmesh/macros/macros.sql

@@ -1,8 +0,0 @@
-{% macro alias(expr, alias) %}
-  {{ expr }} AS {{ alias }}
-{% endmacro %}
-
-
-{% macro identity(y) %}
-  {{ y }}
-{% endmacro %}

package/recce-source/integration_tests/sqlmesh/macros/utils.py

@@ -1,11 +0,0 @@
-from sqlmesh import macro
-
-
-@macro()
-def add_one(evaluator, column):
-    return f"{column} + 1"
-
-
-@macro()
-def between(evaluator, value, start, end):
-    return value.between(start, end)

package/recce-source/integration_tests/sqlmesh/metrics/metrics.sql

@@ -1,25 +0,0 @@
-METRIC (
-  name total_ordered_items,
-  owner jen,
-  expression SUM(sushi.order_items.quantity)
-);
-
-METRIC (
-  name total_orders,
-  owner jen,
-  expression COUNT(sushi.orders.id)
-);
-
-
-METRIC (
-  name total_orders_from_active_customers,
-  owner jen,
-  expression COUNT(IF(sushi.customers.status = 'ACTIVE', sushi.orders.id, NULL))
-);
-
-
-METRIC (
-  name items_per_order,
-  owner jen,
-  expression total_ordered_items / total_orders
-);

package/recce-source/integration_tests/sqlmesh/models/customer_revenue_by_day.sql

@@ -1,41 +0,0 @@
-/* Table of revenue from customers by day. */
-MODEL (
-  name sushi.customer_revenue_by_day,
-  kind incremental_by_time_range (
-    time_column event_date,
-    batch_size 10,
-  ),
-  owner jen,
-  cron '@daily',
-  dialect hive,
-  tags expensive,
-  grain (customer_id, event_date),
-);
-
-WITH order_total AS (
-  SELECT
-    oi.order_id AS order_id,
-    SUM(oi.quantity * i.price) AS total,
-    oi.event_date AS event_date
-  FROM sushi.order_items AS oi
-  LEFT JOIN sushi.items AS i
-    ON oi.item_id = i.id AND oi.event_date = i.event_date
-  WHERE
-    oi.event_date BETWEEN CAST('{{ start_ds }}' as DATE) AND CAST('{{ end_ds }}' as DATE)
-  GROUP BY
-    oi.order_id,
-    oi.event_date
-)
-SELECT
-  o.customer_id::INT AS customer_id, /* Customer id */
-  SUM(ot.total)::DOUBLE AS revenue, /* Revenue from orders made by this customer */
-  MAX(0) AS "country code", /* Customer country code, used for testing spaces */
-  o.event_date::DATE AS event_date /* Date */
-FROM sushi.orders AS o
-LEFT JOIN order_total AS ot
-  ON o.id = ot.order_id AND o.event_date = ot.event_date
-WHERE
-  o.event_date BETWEEN CAST('{{ start_ds }}' as DATE) AND CAST('{{ end_ds }}' as DATE)
-GROUP BY
-  o.customer_id,
-  o.event_date

package/recce-source/integration_tests/sqlmesh/models/customer_revenue_lifetime.sql

@@ -1,60 +0,0 @@
-/*
-    Table of lifetime customer revenue.
-    Date is available to get lifetime value up to a certain date.
-    Use latest date to get current lifetime value.
-*/
-MODEL (
-  name sushi.customer_revenue_lifetime,
-  kind incremental_by_time_range (
-    time_column event_date,
-    batch_size 1
-  ),
-  owner jen,
-  cron '@daily',
-  dialect hive,
-  tags expensive,
-  columns (
-    customer_id INT,
-    revenue DOUBLE,
-    event_date DATE
-  ),
-  grain (customer_id, event_date),
-);
-
-WITH order_total AS (
-  SELECT
-    oi.order_id AS order_id,
-    SUM(oi.quantity * i.price) AS total
-  FROM sushi.order_items AS oi
-  LEFT JOIN sushi.items AS i
-    ON oi.item_id = i.id AND oi.event_date = i.event_date
-  WHERE
-    oi.event_date = @end_date
-  GROUP BY
-    oi.order_id
-), incremental_total AS (
-  SELECT
-    o.customer_id::INT AS customer_id,
-    SUM(ot.total)::DOUBLE AS revenue,
-  FROM sushi.orders AS o
-  LEFT JOIN order_total AS ot
-    ON o.id = ot.order_id
-  WHERE
-    o.event_date = @end_date
-  GROUP BY
-    o.customer_id
-), prev_total AS (
-  SELECT
-    crl.customer_id,
-    crl.revenue
-  FROM sushi.customer_revenue_lifetime AS crl
-  WHERE
-    crl.event_date = @end_date - INTERVAL 1 DAY
-)
-SELECT
-  COALESCE(it.customer_id, prev_total.customer_id) AS customer_id, /* Customer id */
-  COALESCE(it.revenue, 0) + COALESCE(prev_total.revenue, 0) AS revenue, /* Lifetime revenue from this customer */
-  @end_date AS event_date /* End date of the lifetime calculation */
-FROM incremental_total AS it
-FULL OUTER JOIN prev_total AS prev_total
-  ON it.customer_id = prev_total.customer_id

package/recce-source/integration_tests/sqlmesh/models/customers.sql

@@ -1,32 +0,0 @@
-MODEL (
-  name sushi.customers,
-  kind FULL,
-  owner jen,
-  cron '@daily',
-  tags (pii, fact),
-  grain customer_id,
-  description 'Sushi customer data'
-);
-
-CREATE SCHEMA IF NOT EXISTS raw;
-DROP TABLE IF EXISTS raw.demographics;
-CREATE TABLE raw.demographics AS (
-  SELECT 1 AS customer_id, '00000' AS zip
-);
-
-WITH current_marketing AS (
-  SELECT
-    customer_id,
-    status
-  FROM sushi.marketing
-  WHERE valid_to is null
-)
-SELECT DISTINCT
-  o.customer_id::INT AS customer_id, -- customer_id uniquely identifies customers
-  m.status,
-  d.zip
-  FROM sushi.orders AS o
-LEFT JOIN current_marketing AS m
-  ON o.customer_id = m.customer_id
-  LEFT JOIN raw.demographics AS d
-  ON o.customer_id = d.customer_id

package/recce-source/integration_tests/sqlmesh/models/items.py

@@ -1,95 +0,0 @@
-import random
-import typing as t
-from datetime import datetime
-
-import numpy as np
-import pandas as pd
-from helper import iter_dates  # type: ignore
-from sqlglot.expressions import to_column
-
-from sqlmesh import ExecutionContext, model
-from sqlmesh.core.model import IncrementalByTimeRangeKind
-from sqlmesh.utils.date import to_date
-
-ITEMS = [
-    "Ahi",
-    "Aji",
-    "Amaebi",
-    "Anago",
-    "Aoyagi",
-    "Bincho",
-    "Katsuo",
-    "Ebi",
-    "Escolar",
-    "Hamachi",
-    "Hamachi Toro",
-    "Hirame",
-    "Hokigai",
-    "Hotate",
-    "Ika",
-    "Ikura",
-    "Iwashi",
-    "Kani",
-    "Kanpachi",
-    "Maguro",
-    "Saba",
-    "Sake",
-    "Sake Toro",
-    "Tai",
-    "Tako",
-    "Tamago",
-    "Tobiko",
-    "Toro",
-    "Tsubugai",
-    "Umi Masu",
-    "Unagi",
-    "Uni",
-]
-
-
-@model(
-    "sushi.items",
-    kind=IncrementalByTimeRangeKind(time_column="event_date", batch_size=30),
-    start="1 week ago",
-    cron="@daily",
-    columns={
-        "id": "int",
-        "name": "text",
-        "price": "double",
-        "event_date": "date",
-    },
-    audits=[
-        ("accepted_values", {"column": to_column("name"), "is_in": ITEMS}),
-        ("not_null", {"columns": [to_column("name"), to_column("price")]}),
-        ("assert_items_price_exceeds_threshold", {"price": 0}),
-    ],
-    session_properties={
-        "string_prop": "some_value",
-        "int_prop": 1,
-        "float_prop": 1.0,
-        "bool_prop": True,
-    },
-)
-def execute(
-    context: ExecutionContext,
-    start: datetime,
-    end: datetime,
-    execution_time: datetime,
-    **kwargs: t.Any,
-) -> pd.DataFrame:
-    dfs = []
-    for dt in iter_dates(start, end):
-        num_items = random.randint(10, len(ITEMS))
-        dfs.append(
-            pd.DataFrame(
-                {
-                    "name": random.sample(ITEMS, num_items),
-                    "price": np.random.uniform(3.0, 10.0, size=num_items).round(2),
-                    "event_date": to_date(dt),
-                }
-            )
-            .reset_index()
-            .rename(columns={"index": "id"})
-        )
-
-    return pd.concat(dfs)

package/recce-source/integration_tests/sqlmesh/models/marketing.sql

@@ -1,15 +0,0 @@
-MODEL (
-  name sushi.marketing,
-  kind SCD_TYPE_2(unique_key customer_id),
-  owner jen,
-  cron '@daily',
-  grain customer_id,
-  description 'Sushi marketing data'
-);
-
-SELECT
-    customer_id::INT AS customer_id, -- customer_id uniquely identifies customers \
-    status::TEXT AS status,
-    updated_at::TIMESTAMP AS updated_at
-FROM
-    sushi.raw_marketing