@datarecce/ui 0.1.41 → 0.2.0

package/recce-source/integration_tests/sqlmesh/models/order_items.py

@@ -1,95 +0,0 @@
-import random
-import typing as t
-from datetime import datetime
-
-import numpy as np
-import pandas as pd
-from helper import iter_dates  # type: ignore
-from sqlglot import exp
-from sqlglot.expressions import to_column
-
-from sqlmesh import ExecutionContext, model
-from sqlmesh.core.model import IncrementalByTimeRangeKind
-from sqlmesh.utils.date import to_date, to_ds
-
-ITEMS = "sushi.items"
-
-
-def get_items_table(context: ExecutionContext) -> str:
-    return context.table(ITEMS)
-
-
-@model(
-    "sushi.order_items",
-    kind=IncrementalByTimeRangeKind(
-        time_column="event_date",
-        batch_size=30,
-    ),
-    cron="@daily",
-    columns={
-        "id": "int",
-        "order_id": "int",
-        "item_id": "int",
-        "quantity": "int",
-        "event_date": "date",
-    },
-    audits=[
-        (
-            "NOT_NULL",
-            {"columns": [to_column(c) for c in ("id", "order_id", "item_id", "quantity")]},
-        ),
-        ("assert_order_items_quantity_exceeds_threshold", {"quantity": 0}),
-    ],
-)
-def execute(
-    context: ExecutionContext,
-    start: datetime,
-    end: datetime,
-    execution_time: datetime,
-    **kwargs: t.Any,
-) -> t.Generator[pd.DataFrame, None, None]:
-    orders_table = context.table("sushi.orders")
-    items_table = get_items_table(context)
-
-    rng = random.Random()
-
-    for dt in iter_dates(start, end):
-        rng.seed(int(dt.timestamp()))
-
-        # Generate query with sqlglot dialect/quoting
-        orders = context.fetchdf(
-            exp.select("*").from_(orders_table).where(f"event_date = CAST('{to_ds(dt)}' AS DATE)"),
-            quote_identifiers=True,
-        )
-
-        # Normalize column names to support Snowflake.
-        orders = orders.rename(columns={col: col.lower() for col in orders.columns})  # type: ignore
-
-        # Generate query with sqlglot dialect/quoting
-        items = context.fetchdf(
-            exp.select("*").from_(items_table).where(f"event_date = CAST('{to_ds(dt)}' AS DATE)"),
-            quote_identifiers=True,
-        )
-
-        # Normalize column names to support Snowflake.
-        items = items.rename(columns={col: col.lower() for col in items.columns})  # type: ignore
-
-        dfs = []
-
-        for order_id in orders["id"]:
-            n = rng.randint(1, 5)
-
-            dfs.append(
-                pd.DataFrame(
-                    {
-                        "order_id": order_id,
-                        "item_id": items.sample(n=n)["id"],
-                        "quantity": np.random.randint(1, 10, n),
-                        "event_date": to_date(dt),
-                    }
-                )
-                .reset_index()
-                .rename(columns={"index": "id"})
-            )
-
-        yield pd.concat(dfs)

package/recce-source/integration_tests/sqlmesh/models/orders.py

@@ -1,70 +0,0 @@
-import random
-import typing as t
-from datetime import datetime, timedelta
-
-import pandas as pd
-from helper import iter_dates  # type: ignore
-
-from sqlmesh import ExecutionContext, model
-from sqlmesh.core.model import IncrementalByTimeRangeKind
-from sqlmesh.utils.date import to_date
-
-CUSTOMERS = list(range(0, 100))
-WAITERS = list(range(0, 10))
-
-
-@model(
-    "sushi.orders",
-    description="Table of sushi orders.",
-    kind=IncrementalByTimeRangeKind(time_column="event_date", batch_size=30),
-    start="1 week ago",
-    cron="@daily",
-    grains=[
-        "id AS order_id",
-    ],
-    references=[
-        "customer_id",
-        "waiter_id",
-    ],
-    columns={
-        "id": "int",
-        "customer_id": "int",
-        "waiter_id": "int",
-        "start_ts": "int",
-        "end_ts": "int",
-        "event_date": "date",
-    },
-)
-def execute(
-    context: ExecutionContext,
-    start: datetime,
-    end: datetime,
-    execution_time: datetime,
-    **kwargs: t.Any,
-) -> pd.DataFrame:
-    dfs = []
-    for dt in iter_dates(start, end):
-        num_orders = random.randint(10, 30)
-
-        start_ts = [
-            int((dt + timedelta(seconds=random.randint(0, 80000))).timestamp())
-            for _ in range(num_orders)
-        ]
-
-        end_ts = [int(s + random.randint(0, 60 * 60)) for s in start_ts]
-
-        dfs.append(
-            pd.DataFrame(
-                {
-                    "customer_id": random.choices(CUSTOMERS, k=num_orders),
-                    "waiter_id": random.choices(WAITERS, k=num_orders),
-                    "start_ts": start_ts,
-                    "end_ts": end_ts,
-                    "event_date": to_date(dt),
-                }
-            )
-            .reset_index()
-            .rename(columns={"index": "id"})
-        )
-
-    return pd.concat(dfs)

package/recce-source/integration_tests/sqlmesh/models/raw_marketing.py

@@ -1,62 +0,0 @@
-import random
-import typing as t
-from datetime import datetime
-
-import numpy as np
-import pandas as pd
-from helper import iter_dates  # noqa
-from sqlglot import exp
-from sqlmesh import ExecutionContext, model
-from sqlmesh.core.model import FullKind
-
-
-@model(
-    "sushi.raw_marketing",
-    description="Table of marketing status.",
-    kind=FullKind(),
-    start="1 week ago",
-    cron="@daily",
-    grains=[
-        "customer_id",
-    ],
-    columns={
-        "customer_id": "int",
-        "status": "text",
-        "updated_at": "timestamp",
-    },
-)
-def execute(
-    context: ExecutionContext,
-    end: datetime,
-    execution_time: datetime,
-    **kwargs: t.Any,
-) -> pd.DataFrame:
-    # Generate query with sqlglot dialect/quoting
-    existing_table = context.table("sushi.raw_marketing")
-    df_existing = context.fetchdf(
-        exp.select("customer_id", "status", "updated_at").from_(existing_table),
-        quote_identifiers=True,
-    )
-
-    seed = int(end.strftime("%Y%m%d"))
-    np.random.seed(seed)
-    num_customers = random.randint(30, 100)
-
-    # Remove timezone because we specify `timestamp` in the columns type list
-    exec_time = execution_time.replace(tzinfo=None)
-
-    df_new = pd.DataFrame(
-        {
-            "customer_id": random.sample(range(0, 100), k=num_customers),
-            "status": np.random.choice(["active", "inactive"], size=num_customers, p=[0.8, 0.2]),
-            "updated_at": [exec_time] * num_customers,
-        }
-    )
-    df = df_new.merge(df_existing, on="customer_id", how="left", suffixes=(None, "_old"))
-    df["updated_at"] = pd.to_datetime(
-        np.where(df["status_old"] != df["status"], execution_time, df["updated_at_old"]),  # type: ignore
-        errors="coerce",
-        utc=True,
-    )
-    df = df.drop(columns=["status_old", "updated_at_old"])
-    return df

package/recce-source/integration_tests/sqlmesh/models/top_waiters.sql

@@ -1,23 +0,0 @@
-/* View of top waiters. */
-MODEL (
-  name sushi.top_waiters,
-  owner jen,
-  audits (
-    unique_values(columns=(waiter_id))
-  ),
-  grain waiter_id
-);
-
-SELECT
-  waiter_id::INT AS waiter_id,
-  revenue::DOUBLE AS revenue
-FROM sushi.waiter_revenue_by_day
-WHERE
-  event_date = (
-    SELECT
-      MAX(event_date)
-    FROM sushi.waiter_revenue_by_day
-  )
-ORDER BY
-  revenue DESC
-LIMIT 10

package/recce-source/integration_tests/sqlmesh/models/waiter_as_customer_by_day.sql

@@ -1,29 +0,0 @@
-
-MODEL (
-  name sushi.waiter_as_customer_by_day,
-  kind incremental_by_time_range (
-    time_column event_date
-  ),
-  owner jen,
-  cron '@daily',
-  audits (
-    not_null(columns = (waiter_id)),
-    forall(criteria = (LENGTH(waiter_name) > 0))
-  )
-);
-
-JINJA_QUERY_BEGIN;
-
-{% set x = 1 %}
-
-SELECT
-  w.waiter_id as waiter_id,
-  wn.name as waiter_name,
-  {{ alias(identity(x), 'flag') }},
-  w.event_date as event_date
-FROM sushi.waiters AS w
-JOIN sushi.customers as c ON w.waiter_id = c.customer_id
-JOIN sushi.waiter_names as wn ON w.waiter_id = wn.id
-WHERE w.event_date BETWEEN @start_date AND @end_date;
-
-JINJA_END;

package/recce-source/integration_tests/sqlmesh/models/waiter_names.sql

@@ -1,10 +0,0 @@
-MODEL (
-  name sushi.waiter_names,
-  kind SEED (
-    path '../seeds/waiter_names.csv',
-    batch_size 5
-  ),
-  owner jen,
-  grain id,
-  description 'List of waiter names'
-)

package/recce-source/integration_tests/sqlmesh/models/waiter_revenue_by_day.sql

@@ -1,29 +0,0 @@
-/* Table of revenue generated by waiters by day. */
-MODEL (
-  name sushi.waiter_revenue_by_day,
-  kind incremental_by_time_range (
-    time_column event_date,
-    batch_size 10,
-  ),
-  owner jen,
-  cron '@daily',
-  audits (
-    NUMBER_OF_ROWS(threshold=0)
-  ),
-  grain (waiter_id, event_date)
-);
-
-SELECT
-  o.waiter_id::INT AS waiter_id, /* Waiter id */
-  SUM(oi.quantity * i.price)::DOUBLE AS revenue, /* Revenue from orders taken by this waiter */
-  o.event_date::DATE AS event_date /* Date */
-FROM sushi.orders AS o
-LEFT JOIN sushi.order_items AS oi
-  ON o.id = oi.order_id AND o.event_date = oi.event_date
-LEFT JOIN sushi.items AS i
-  ON oi.item_id = i.id AND oi.event_date = i.event_date
-WHERE
-  o.event_date BETWEEN @start_date AND @end_date
-GROUP BY
-  o.waiter_id,
-  o.event_date

package/recce-source/integration_tests/sqlmesh/models/waiters.py

@@ -1,62 +0,0 @@
-from macros.macros import incremental_by_ds  # type: ignore
-from sqlglot import exp
-
-from sqlmesh import model
-from sqlmesh.core.macros import MacroEvaluator
-from sqlmesh.core.model import EmbeddedKind
-
-
-@model(
-    "sushi.waiters",
-    is_sql=True,
-    kind=EmbeddedKind(),
-    owner="jen",
-    cron="@daily",
-)
-def entrypoint(evaluator: MacroEvaluator) -> exp.Select:
-    """
-    This creates the following model using SQLGlot's builder methods:
-
-    MODEL (
-      name sushi.waiters,
-      kind EMBEDDED,
-      owner jen,
-      cron '@daily',
-    );
-
-    SELECT DISTINCT
-      waiter_id::INT AS waiter_id,
-      event_date::DATE AS event_date
-    FROM sushi.orders AS o
-    WHERE @incremental_by_ds(ds)
-    """
-    if evaluator.runtime_stage != "loading":
-        snapshot = evaluator.get_snapshot("sushi.waiters")
-        assert snapshot is not None
-
-        parent_snapshots = snapshot.parents
-        assert len(parent_snapshots) == 1
-        name = '"sushi"."orders"'
-        # There are tests which force not having a default catalog so we check here if one is defined
-        # and add it to the name if it is
-        default_catalog = evaluator.default_catalog
-        if default_catalog:
-            # make sure we don't double quote the default catalog
-            default_catalog = default_catalog.strip('"')
-            name = ".".join([f'"{default_catalog}"', name])
-        assert (
-            parent_snapshots[0].name == name
-        ), f"Snapshot Name: {parent_snapshots[0].name}, Name: {name}"
-
-    excluded = {"id", "customer_id", "start_ts", "end_ts"}
-    projections = []
-    for column, dtype in evaluator.columns_to_types("sushi.orders").items():
-        if column not in excluded:
-            projections.append(f"{column}::{dtype}")
-
-    return (
-        exp.select(*projections)
-        .from_("sushi.orders AS o")
-        .where(incremental_by_ds(evaluator, exp.to_column("event_date")))
-        .distinct()
-    )

package/recce-source/integration_tests/sqlmesh/prep_env.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -euxo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-cd "$SCRIPT_DIR"
-pwd
-
-# Step 1: prepare env
-sqlmesh --version
-sqlmesh --config local_config plan --no-prompts --auto-apply
-sqlmesh --config local_config plan dev --no-prompts --auto-apply --include-unmodified
-
-# Step 2: turn off anonymous tracking
-mkdir ~/.recce
-echo "user_id: 00000000000000000000000000000000" > ~/.recce/profile.yml
-echo "anonymous_tracking: false" >> ~/.recce/profile.yml

package/recce-source/integration_tests/sqlmesh/schema.yaml

@@ -1,5 +0,0 @@
-- name: raw.demographics
-  description: Table containing demographics information
-  columns:
-    customer_id: int
-    zip: text

package/recce-source/integration_tests/sqlmesh/seeds/waiter_names.csv

@@ -1,11 +0,0 @@
-id,name
-0,Toby
-1,Tyson
-2,Ryan
-3,George
-4,Chris
-5,Max
-6,Vincent
-7,Iaroslav
-8,Emma
-9,Maia

package/recce-source/integration_tests/sqlmesh/test_server.sh

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-set -euxo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-cd "$SCRIPT_DIR"
-pwd
-
-echo "Starting the server..."
-recce server --sqlmesh  --sqlmesh-envs prod:dev --sqlmesh-config local_config &
-
-echo "Waiting for the server to respond..."
-timeout 20 bash -c 'until curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/api/info | grep -q 200; do
-  echo "Server not ready yet..."
-  sleep 2
-done'
-
-if [ $? -eq 0 ]; then
-  echo "Server is up and running."
-  EXITCODE=0
-else
-  echo "Failed to start the server within the time limit."
-  EXITCODE=1
-fi
-
-echo "Stopping the server..."
-kill $(jobs -p) || true
-echo "Server stopped."
-
-exit $EXITCODE

package/recce-source/integration_tests/sqlmesh/tests/test_customer_revenue_by_day.yaml

@@ -1,63 +0,0 @@
-test_customer_revenue_by_day:
-  model: sushi.customer_revenue_by_day
-  inputs:
-    sushi.orders:
-      - id: 1
-        customer_id: 1
-        waiter_id: 1
-        start_ts: 1641002340
-        end_ts: 1641004140
-        event_date: 2022-01-01
-      - id: 2
-        customer_id: 1
-        waiter_id: 2
-        start_ts: 1641007740
-        end_ts: 1641009540
-        event_date: 2022-01-01
-    sushi.order_items:
-      - id: 1
-        order_id: 1
-        item_id: 1
-        quantity: 2
-        event_date: 2022-01-01
-      - id: 2
-        order_id: 1
-        item_id: 2
-        quantity: 3
-        event_date: 2022-01-01
-      - id: 3
-        order_id: 2
-        item_id: 1
-        quantity: 4
-        event_date: 2022-01-01
-      - id: 4
-        order_id: 2
-        item_id: 2
-        quantity: 5
-        event_date: 2022-01-01
-    sushi.items:
-      - id: 1
-        name: maguro
-        price: 1.23
-        event_date: 2022-01-01
-      - id: 2
-        name: ika
-        price: 2.34
-        event_date: 2022-01-01
-  outputs:
-    ctes:
-      order_total:
-        - order_id: 1
-          total: 9.48
-          event_date: 2022-01-01
-        - order_id: 2
-          total: 16.62
-          event_date: 2022-01-01
-    query:
-      - customer_id: 1
-        revenue: 26.1
-        country code: 0
-        event_date: 2022-01-01
-  vars:
-    start: 2022-01-01
-    end: 2022-01-01

package/recce-source/integration_tests/sqlmesh/tests/test_order_items.yaml

@@ -1,72 +0,0 @@
-test_order_items:
-  model: sushi.order_items
-  inputs:
-    sushi.orders:
-      - id: 1
-        customer_id: 1
-        waiter_id: 1
-        start_ts: 1641002340
-        end_ts: 1641004140
-        event_date: 2022-01-01
-      - id: 2
-        customer_id: 1
-        waiter_id: 2
-        start_ts: 1641007740
-        end_ts: 1641009540
-        event_date: 2022-01-01
-    sushi.items:
-      - id: 1
-        name: ahi
-        price: 1.23
-        event_date: 2022-01-01
-      - id: 2
-        name: aji
-        price: 2.34
-        event_date: 2022-01-01
-      - id: 3
-        name: amaebi
-        price: 2.34
-        event_date: 2022-01-01
-      - id: 4
-        name: anago
-        price: 2.34
-        event_date: 2022-01-01
-      - id: 5
-        name: aoyagi
-        price: 2.34
-        event_date: 2022-01-01
-  outputs:
-    query:
-      - id: 1
-        order_id: 1
-        item_id: 2
-        quantity: 4
-        event_date: 2022-01-01
-      - id: 2
-        order_id: 2
-        item_id: 3
-        quantity: 7
-        event_date: 2022-01-01
-      - id: 1
-        order_id: 2
-        item_id: 2
-        quantity: 8
-        event_date: 2022-01-01
-      - id: 0
-        order_id: 2
-        item_id: 1
-        quantity: 9
-        event_date: 2022-01-01
-      - id: 4
-        order_id: 2
-        item_id: 5
-        quantity: 9
-        event_date: 2022-01-01
-      - id: 3
-        order_id: 2
-        item_id: 4
-        quantity: 5
-        event_date: 2022-01-01
-  vars:
-    start: 2022-01-01
-    end: 2022-01-01

package/recce-source/js/.editorconfig

@@ -1,27 +0,0 @@
-# EditorConfig is awesome: http://EditorConfig.org
-
-# top-most EditorConfig file
-root = true
-
-# Unix-style newlines with a newline ending every file
-[*]
-end_of_line = lf
-insert_final_newline = true
-trim_trailing_whitespace = true
-charset = utf-8
-
-# 2 space indentation
-[*.{js,jsx,json,yml,yaml,html,cwl,tsx,ts}]
-indent_style = space
-indent_size = 2
-ij_javascript_force_semicolon_style = true
-ij_typescript_force_semicolon_style = true
-ij_typescript_use_double_quotes = true
-ij_javascript_use_double_quotes = true
-ij_javascript_force_quote_style = true
-ij_typescript_force_quote_style = true
-
-[*.{md,Rmd,rst}]
-trim_trailing_whitespace = false
-indent_style = space
-indent_size = 2

package/recce-source/js/.env.development

@@ -1,5 +0,0 @@
-# .env.development
-NEXT_PUBLIC_API_URL=http://localhost:8000
-NEXT_PUBLIC_CLOUD_WEB_URL=http://localhost:3000
-AMPLITUDE_API_KEY=
-GTM_ID=

package/recce-source/js/.husky/pre-commit

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Check if there are any staged TypeScript/JavaScript files in the js directory
-staged_js_files=$(git diff --cached --name-only | grep "^js/" | grep -E '\.(ts|tsx|js|jsx)$' || true)
-
-if [ -n "$staged_js_files" ]; then
-	echo "TypeScript/JavaScript files changed, running lint and type check..."
-	(cd js && pnpm lint:staged && pnpm type:check && pnpm test)
-else
-	echo "No TypeScript/JavaScript files changed, skipping lint and type check."
-fi
-
-# start templated
-INSTALL_PYTHON=${INSTALL_PYTHON:-$(command -v python || true)}
-ARGS="hook-impl --config=.pre-commit-config.yaml --hook-type=pre-commit"
-# end templated
-
-HERE="$(cd "$(dirname "$0")" && pwd)"
-STRIP_JS_HUSKY=$(echo "$HERE" | sed 's|/js/\.husky$||')
-ARGS="$ARGS --hook-dir $STRIP_JS_HUSKY -- $*"
-
-if [ -x "$INSTALL_PYTHON" ] && "$INSTALL_PYTHON" -c "import pre_commit" 2>/dev/null; then
-	exec "$INSTALL_PYTHON" -mpre_commit $ARGS
-elif command -v pre-commit >/dev/null; then
-	exec pre-commit $ARGS
-else
-	echo '`pre-commit` not found.  Did you forget to activate your virtualenv?' 1>&2
-	echo "Skipping pre-commit hook."
-fi

package/recce-source/js/.nvmrc

@@ -1 +0,0 @@
-24.9.0