@datarecce/ui 0.1.30 → 0.1.32

package/recce-source/integration_tests/sqlmesh/models/raw_marketing.py

@@ -0,0 +1,62 @@
+import random
+import typing as t
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+from helper import iter_dates  # noqa
+from sqlglot import exp
+from sqlmesh import ExecutionContext, model
+from sqlmesh.core.model import FullKind
+
+
+@model(
+    "sushi.raw_marketing",
+    description="Table of marketing status.",
+    kind=FullKind(),
+    start="1 week ago",
+    cron="@daily",
+    grains=[
+        "customer_id",
+    ],
+    columns={
+        "customer_id": "int",
+        "status": "text",
+        "updated_at": "timestamp",
+    },
+)
+def execute(
+    context: ExecutionContext,
+    end: datetime,
+    execution_time: datetime,
+    **kwargs: t.Any,
+) -> pd.DataFrame:
+    # Generate query with sqlglot dialect/quoting
+    existing_table = context.table("sushi.raw_marketing")
+    df_existing = context.fetchdf(
+        exp.select("customer_id", "status", "updated_at").from_(existing_table),
+        quote_identifiers=True,
+    )
+
+    seed = int(end.strftime("%Y%m%d"))
+    np.random.seed(seed)
+    num_customers = random.randint(30, 100)
+
+    # Remove timezone because we specify `timestamp` in the columns type list
+    exec_time = execution_time.replace(tzinfo=None)
+
+    df_new = pd.DataFrame(
+        {
+            "customer_id": random.sample(range(0, 100), k=num_customers),
+            "status": np.random.choice(["active", "inactive"], size=num_customers, p=[0.8, 0.2]),
+            "updated_at": [exec_time] * num_customers,
+        }
+    )
+    df = df_new.merge(df_existing, on="customer_id", how="left", suffixes=(None, "_old"))
+    df["updated_at"] = pd.to_datetime(
+        np.where(df["status_old"] != df["status"], execution_time, df["updated_at_old"]),  # type: ignore
+        errors="coerce",
+        utc=True,
+    )
+    df = df.drop(columns=["status_old", "updated_at_old"])
+    return df

package/recce-source/integration_tests/sqlmesh/models/top_waiters.sql

@@ -0,0 +1,23 @@
+/* View of top waiters. */
+MODEL (
+  name sushi.top_waiters,
+  owner jen,
+  audits (
+    unique_values(columns=(waiter_id))
+  ),
+  grain waiter_id
+);
+
+SELECT
+  waiter_id::INT AS waiter_id,
+  revenue::DOUBLE AS revenue
+FROM sushi.waiter_revenue_by_day
+WHERE
+  event_date = (
+    SELECT
+      MAX(event_date)
+    FROM sushi.waiter_revenue_by_day
+  )
+ORDER BY
+  revenue DESC
+LIMIT 10

package/recce-source/integration_tests/sqlmesh/models/waiter_as_customer_by_day.sql

@@ -0,0 +1,29 @@
+
+MODEL (
+  name sushi.waiter_as_customer_by_day,
+  kind incremental_by_time_range (
+    time_column event_date
+  ),
+  owner jen,
+  cron '@daily',
+  audits (
+    not_null(columns = (waiter_id)),
+    forall(criteria = (LENGTH(waiter_name) > 0))
+  )
+);
+
+JINJA_QUERY_BEGIN;
+
+{% set x = 1 %}
+
+SELECT
+  w.waiter_id as waiter_id,
+  wn.name as waiter_name,
+  {{ alias(identity(x), 'flag') }},
+  w.event_date as event_date
+FROM sushi.waiters AS w
+JOIN sushi.customers as c ON w.waiter_id = c.customer_id
+JOIN sushi.waiter_names as wn ON w.waiter_id = wn.id
+WHERE w.event_date BETWEEN @start_date AND @end_date;
+
+JINJA_END;

package/recce-source/integration_tests/sqlmesh/models/waiter_names.sql

@@ -0,0 +1,10 @@
+MODEL (
+  name sushi.waiter_names,
+  kind SEED (
+    path '../seeds/waiter_names.csv',
+    batch_size 5
+  ),
+  owner jen,
+  grain id,
+  description 'List of waiter names'
+)

package/recce-source/integration_tests/sqlmesh/models/waiter_revenue_by_day.sql

@@ -0,0 +1,29 @@
+/* Table of revenue generated by waiters by day. */
+MODEL (
+  name sushi.waiter_revenue_by_day,
+  kind incremental_by_time_range (
+    time_column event_date,
+    batch_size 10,
+  ),
+  owner jen,
+  cron '@daily',
+  audits (
+    NUMBER_OF_ROWS(threshold=0)
+  ),
+  grain (waiter_id, event_date)
+);
+
+SELECT
+  o.waiter_id::INT AS waiter_id, /* Waiter id */
+  SUM(oi.quantity * i.price)::DOUBLE AS revenue, /* Revenue from orders taken by this waiter */
+  o.event_date::DATE AS event_date /* Date */
+FROM sushi.orders AS o
+LEFT JOIN sushi.order_items AS oi
+  ON o.id = oi.order_id AND o.event_date = oi.event_date
+LEFT JOIN sushi.items AS i
+  ON oi.item_id = i.id AND oi.event_date = i.event_date
+WHERE
+  o.event_date BETWEEN @start_date AND @end_date
+GROUP BY
+  o.waiter_id,
+  o.event_date

package/recce-source/integration_tests/sqlmesh/models/waiters.py

@@ -0,0 +1,62 @@
+from macros.macros import incremental_by_ds  # type: ignore
+from sqlglot import exp
+
+from sqlmesh import model
+from sqlmesh.core.macros import MacroEvaluator
+from sqlmesh.core.model import EmbeddedKind
+
+
+@model(
+    "sushi.waiters",
+    is_sql=True,
+    kind=EmbeddedKind(),
+    owner="jen",
+    cron="@daily",
+)
+def entrypoint(evaluator: MacroEvaluator) -> exp.Select:
+    """
+    This creates the following model using SQLGlot's builder methods:
+
+    MODEL (
+      name sushi.waiters,
+      kind EMBEDDED,
+      owner jen,
+      cron '@daily',
+    );
+
+    SELECT DISTINCT
+      waiter_id::INT AS waiter_id,
+      event_date::DATE AS event_date
+    FROM sushi.orders AS o
+    WHERE @incremental_by_ds(ds)
+    """
+    if evaluator.runtime_stage != "loading":
+        snapshot = evaluator.get_snapshot("sushi.waiters")
+        assert snapshot is not None
+
+        parent_snapshots = snapshot.parents
+        assert len(parent_snapshots) == 1
+        name = '"sushi"."orders"'
+        # There are tests which force not having a default catalog so we check here if one is defined
+        # and add it to the name if it is
+        default_catalog = evaluator.default_catalog
+        if default_catalog:
+            # make sure we don't double quote the default catalog
+            default_catalog = default_catalog.strip('"')
+            name = ".".join([f'"{default_catalog}"', name])
+        assert (
+            parent_snapshots[0].name == name
+        ), f"Snapshot Name: {parent_snapshots[0].name}, Name: {name}"
+
+    excluded = {"id", "customer_id", "start_ts", "end_ts"}
+    projections = []
+    for column, dtype in evaluator.columns_to_types("sushi.orders").items():
+        if column not in excluded:
+            projections.append(f"{column}::{dtype}")
+
+    return (
+        exp.select(*projections)
+        .from_("sushi.orders AS o")
+        .where(incremental_by_ds(evaluator, exp.to_column("event_date")))
+        .distinct()
+    )

package/recce-source/integration_tests/sqlmesh/prep_env.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+pwd
+
+# Step 1: prepare env
+sqlmesh --version
+sqlmesh --config local_config plan --no-prompts --auto-apply
+sqlmesh --config local_config plan dev --no-prompts --auto-apply --include-unmodified
+
+# Step 2: turn off anonymous tracking
+mkdir ~/.recce
+echo "user_id: 00000000000000000000000000000000" > ~/.recce/profile.yml
+echo "anonymous_tracking: false" >> ~/.recce/profile.yml

package/recce-source/integration_tests/sqlmesh/schema.yaml

@@ -0,0 +1,5 @@
+- name: raw.demographics
+  description: Table containing demographics information
+  columns:
+    customer_id: int
+    zip: text

package/recce-source/integration_tests/sqlmesh/seeds/waiter_names.csv

@@ -0,0 +1,11 @@
+id,name
+0,Toby
+1,Tyson
+2,Ryan
+3,George
+4,Chris
+5,Max
+6,Vincent
+7,Iaroslav
+8,Emma
+9,Maia

package/recce-source/integration_tests/sqlmesh/test_server.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+pwd
+
+echo "Starting the server..."
+recce server --sqlmesh  --sqlmesh-envs prod:dev --sqlmesh-config local_config &
+
+echo "Waiting for the server to respond..."
+timeout 20 bash -c 'until curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/api/info | grep -q 200; do
+  echo "Server not ready yet..."
+  sleep 2
+done'
+
+if [ $? -eq 0 ]; then
+  echo "Server is up and running."
+  EXITCODE=0
+else
+  echo "Failed to start the server within the time limit."
+  EXITCODE=1
+fi
+
+echo "Stopping the server..."
+kill $(jobs -p) || true
+echo "Server stopped."
+
+exit $EXITCODE

package/recce-source/integration_tests/sqlmesh/tests/test_customer_revenue_by_day.yaml

@@ -0,0 +1,63 @@
+test_customer_revenue_by_day:
+  model: sushi.customer_revenue_by_day
+  inputs:
+    sushi.orders:
+      - id: 1
+        customer_id: 1
+        waiter_id: 1
+        start_ts: 1641002340
+        end_ts: 1641004140
+        event_date: 2022-01-01
+      - id: 2
+        customer_id: 1
+        waiter_id: 2
+        start_ts: 1641007740
+        end_ts: 1641009540
+        event_date: 2022-01-01
+    sushi.order_items:
+      - id: 1
+        order_id: 1
+        item_id: 1
+        quantity: 2
+        event_date: 2022-01-01
+      - id: 2
+        order_id: 1
+        item_id: 2
+        quantity: 3
+        event_date: 2022-01-01
+      - id: 3
+        order_id: 2
+        item_id: 1
+        quantity: 4
+        event_date: 2022-01-01
+      - id: 4
+        order_id: 2
+        item_id: 2
+        quantity: 5
+        event_date: 2022-01-01
+    sushi.items:
+      - id: 1
+        name: maguro
+        price: 1.23
+        event_date: 2022-01-01
+      - id: 2
+        name: ika
+        price: 2.34
+        event_date: 2022-01-01
+  outputs:
+    ctes:
+      order_total:
+        - order_id: 1
+          total: 9.48
+          event_date: 2022-01-01
+        - order_id: 2
+          total: 16.62
+          event_date: 2022-01-01
+    query:
+      - customer_id: 1
+        revenue: 26.1
+        country code: 0
+        event_date: 2022-01-01
+  vars:
+    start: 2022-01-01
+    end: 2022-01-01

package/recce-source/integration_tests/sqlmesh/tests/test_order_items.yaml

@@ -0,0 +1,72 @@
+test_order_items:
+  model: sushi.order_items
+  inputs:
+    sushi.orders:
+      - id: 1
+        customer_id: 1
+        waiter_id: 1
+        start_ts: 1641002340
+        end_ts: 1641004140
+        event_date: 2022-01-01
+      - id: 2
+        customer_id: 1
+        waiter_id: 2
+        start_ts: 1641007740
+        end_ts: 1641009540
+        event_date: 2022-01-01
+    sushi.items:
+      - id: 1
+        name: ahi
+        price: 1.23
+        event_date: 2022-01-01
+      - id: 2
+        name: aji
+        price: 2.34
+        event_date: 2022-01-01
+      - id: 3
+        name: amaebi
+        price: 2.34
+        event_date: 2022-01-01
+      - id: 4
+        name: anago
+        price: 2.34
+        event_date: 2022-01-01
+      - id: 5
+        name: aoyagi
+        price: 2.34
+        event_date: 2022-01-01
+  outputs:
+    query:
+      - id: 1
+        order_id: 1
+        item_id: 2
+        quantity: 4
+        event_date: 2022-01-01
+      - id: 2
+        order_id: 2
+        item_id: 3
+        quantity: 7
+        event_date: 2022-01-01
+      - id: 1
+        order_id: 2
+        item_id: 2
+        quantity: 8
+        event_date: 2022-01-01
+      - id: 0
+        order_id: 2
+        item_id: 1
+        quantity: 9
+        event_date: 2022-01-01
+      - id: 4
+        order_id: 2
+        item_id: 5
+        quantity: 9
+        event_date: 2022-01-01
+      - id: 3
+        order_id: 2
+        item_id: 4
+        quantity: 5
+        event_date: 2022-01-01
+  vars:
+    start: 2022-01-01
+    end: 2022-01-01

package/recce-source/js/.editorconfig

@@ -0,0 +1,27 @@
+# EditorConfig is awesome: http://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8
+
+# 2 space indentation
+[*.{js,jsx,json,yml,yaml,html,cwl,tsx,ts}]
+indent_style = space
+indent_size = 2
+ij_javascript_force_semicolon_style = true
+ij_typescript_force_semicolon_style = true
+ij_typescript_use_double_quotes = true
+ij_javascript_use_double_quotes = true
+ij_javascript_force_quote_style = true
+ij_typescript_force_quote_style = true
+
+[*.{md,Rmd,rst}]
+trim_trailing_whitespace = false
+indent_style = space
+indent_size = 2

package/recce-source/js/.env.development

@@ -0,0 +1,5 @@
+# .env.development
+NEXT_PUBLIC_API_URL=http://localhost:8000
+NEXT_PUBLIC_CLOUD_WEB_URL=http://localhost:3000
+AMPLITUDE_API_KEY=
+GTM_ID=

package/recce-source/js/.husky/pre-commit

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Check if there are any staged TypeScript/JavaScript files in the js directory
+staged_js_files=$(git diff --cached --name-only | grep "^js/" | grep -E '\.(ts|tsx|js|jsx)$' || true)
+
+if [ -n "$staged_js_files" ]; then
+	echo "TypeScript/JavaScript files changed, running lint and type check..."
+	(cd js && pnpm lint:staged && pnpm type:check && pnpm test)
+else
+	echo "No TypeScript/JavaScript files changed, skipping lint and type check."
+fi
+
+# start templated
+INSTALL_PYTHON=${INSTALL_PYTHON:-$(command -v python || true)}
+ARGS="hook-impl --config=.pre-commit-config.yaml --hook-type=pre-commit"
+# end templated
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+STRIP_JS_HUSKY=$(echo "$HERE" | sed 's|/js/\.husky$||')
+ARGS="$ARGS --hook-dir $STRIP_JS_HUSKY -- $*"
+
+if [ -x "$INSTALL_PYTHON" ] && "$INSTALL_PYTHON" -c "import pre_commit" 2>/dev/null; then
+	exec "$INSTALL_PYTHON" -mpre_commit $ARGS
+elif command -v pre-commit >/dev/null; then
+	exec pre-commit $ARGS
+else
+	echo '`pre-commit` not found.  Did you forget to activate your virtualenv?' 1>&2
+	echo "Skipping pre-commit hook."
+fi

package/recce-source/js/.nvmrc

@@ -0,0 +1 @@
+24.9.0

package/recce-source/js/README.md

@@ -0,0 +1,39 @@
+## Prerequisites
+
+1. Node (>18)
+   ```
+   nvm use 20
+   ```
+2. pnpm
+
+   ```
+   npm install -g corepack@latest
+   corepack enable
+   corepack install
+   pnpm install
+   ```
+
+## Development
+
+1. Run the python server
+
+   ```bash
+   cd my-dbt-project/
+   recce server
+   ```
+
+   The api server is run at http://localhost:8000/api
+
+2. Run the dev server
+
+   ```bash
+   pnpm dev
+   ```
+
+## Build
+
+```bash
+pnpm build
+cd ../recce/data
+git add .
+```

package/recce-source/js/app/(mainComponents)/DisplayModeToggle.tsx

@@ -0,0 +1,65 @@
+"use client";
+
+import IconButton from "@mui/material/IconButton";
+import Tooltip from "@mui/material/Tooltip";
+import { useTheme } from "next-themes";
+import { useEffect, useState } from "react";
+import { PiMoon, PiSun } from "react-icons/pi";
+
+/**
+ * Display Mode Toggle - switches between light and dark themes
+ *
+ * Uses next-themes to persist the user's preference.
+ * Default is light theme, with future support for system preference.
+ */
+export default function DisplayModeToggle() {
+  const { setTheme, resolvedTheme } = useTheme();
+  const [mounted, setMounted] = useState(false);
+
+  // Avoid hydration mismatch by only rendering after mount
+  useEffect(() => {
+    setMounted(true);
+  }, []);
+
+  const toggleTheme = () => {
+    setTheme(resolvedTheme === "dark" ? "light" : "dark");
+  };
+
+  // Don't render anything until mounted to avoid hydration mismatch
+  if (!mounted) {
+    return (
+      <IconButton
+        size="small"
+        sx={{
+          color: "rgba(255, 255, 255, 0.8)",
+          "&:hover": { bgcolor: "rgba(255, 255, 255, 0.1)" },
+        }}
+        disabled
+      >
+        <PiSun style={{ width: 18, height: 18 }} />
+      </IconButton>
+    );
+  }
+
+  const isDark = resolvedTheme === "dark";
+
+  return (
+    <Tooltip title={isDark ? "Switch to light mode" : "Switch to dark mode"}>
+      <IconButton
+        size="small"
+        onClick={toggleTheme}
+        sx={{
+          color: "rgba(255, 255, 255, 0.8)",
+          "&:hover": { bgcolor: "rgba(255, 255, 255, 0.1)" },
+        }}
+        aria-label={isDark ? "Switch to light mode" : "Switch to dark mode"}
+      >
+        {isDark ? (
+          <PiSun style={{ width: 18, height: 18 }} />
+        ) : (
+          <PiMoon style={{ width: 18, height: 18 }} />
+        )}
+      </IconButton>
+    </Tooltip>
+  );
+}