@datarecce/ui 0.1.29 → 0.1.31

package/recce-source/js/src/utils/formatters.tsx

@@ -0,0 +1,237 @@
+import { formatInTimeZone } from "@jeromefitz/date-fns-tz";
+import { TopKResult } from "@/lib/api/profile";
+
+const NO_VALUE = "-";
+
+/**
+ * "Formatters" -- these are your data formatting that returns a formatted value for UI presentation (e.g. number, string, falsey)
+ */
+
+/**
+ * Source: https://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript
+ * @param bytes
+ * @param decimals
+ * @returns a string that matches nearest byte unit (e.g. kb, mb, gb, etc)
+ */
+export function formatBytes(bytes?: number, decimals = 2) {
+  if (bytes === undefined) return;
+  if (!bytes) return "0 Bytes";
+
+  const k = 1024;
+  const dm = decimals < 0 ? 0 : decimals;
+  const sizes = ["Bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+  return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}`;
+}
+/**
+ * @param dateStr ISO date string
+ * @returns a formatted date string in 'yyyy/MM/dd HH:mm:ss'
+ */
+export function formatReportTime(dateStr?: string) {
+  if (!dateStr) return;
+  const date = new Date(dateStr);
+  const userTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+
+  return formatInTimeZone(date, userTimezone, "yyyy-MM-dd HH:mm:ss zzz");
+}
+
+/**
+ *
+ * @param num number type input
+ * @param locales locale string
+ * @param options
+ * @returns a formatted string number, based on locale & options
+ */
+export function formatNumber(
+  num: number | string | undefined,
+  locales = "en-US",
+  options?: Intl.NumberFormatOptions,
+) {
+  if (typeof num !== "number") return num;
+  return new Intl.NumberFormat(locales, options).format(num);
+}
+
+/**
+ * @param num fractional number type input
+ * @returns a formatted percentage string, based on its percentage proximity to either ends (<0.1% and >99.9%)
+ */
+export function formatIntervalMinMax(num: number) {
+  // *  should show <0.1 % if the value is between (0%, 0.1%]
+  const isLowerBound = num > 0 && num <= 0.001;
+  // *  should show >99.9% if the value is between [99.9%, 100%) .
+  const isUpperBound = num < 1 && num >= 0.999;
+
+  const formatter = (newArg = num) =>
+    formatNumber(newArg, "en-US", {
+      style: "percent",
+      minimumFractionDigits: 1,
+    });
+
+  if (isLowerBound) {
+    const result = formatter(0.001);
+    return `<${result}`;
+  } else if (isUpperBound) {
+    const result = formatter(0.999);
+    return `>${result}`;
+  }
+  return formatter();
+}
+
+// SR side: No need for object record handling
+// CR side: needs record handling
+export function formatTestExpectedOrActual(value?: unknown): unknown {
+  if (!value) {
+    return NO_VALUE;
+  }
+
+  // Needed due to comparison's get assertions DS
+  if (typeof value === "object") {
+    return JSON.stringify(value);
+    // return Object.keys(value).map((key) =>
+    //   typeof value[key] === 'string' ? value[key] : JSON.stringify(value[key]),
+    // );
+  }
+
+  return value;
+}
+
+/**
+  show the most common values (aka Mode)
+  * give null if type invalids
+  * skip null value
+  * show top 2 if the values share the same counting, examples:
+     (more than 2) a:100, b:100, c:100 => a, b, ...
+     (2) a:100, b:100 => a
+     (2) null:100, a:100, b:100 => a, b
+     (2) null:101, a:100, b:100 => a, b
+     (2) a:100, b:100 => a, b
+     (1) a:100 => a
+     (1) a:100, b:99, c:99 => a
+ */
+export function formatTopKMetrics(topK?: TopKResult): {
+  topValues?: string;
+  topCounts?: string;
+} {
+  if (!topK) return {};
+  const { counts, values } = topK;
+  const topValues = `${values[0]}`;
+  const topCounts = `${counts[0]}`;
+
+  return {
+    topValues,
+    topCounts,
+  };
+}
+
+/**
+ * A method to handle falsey non-numbers (relevant for comparison reports with column shifts, where base/target values can be undefined)
+ * @param value {string | number} any value that will be checked as number
+ * @param fn {function} any function to format the valid number
+ * @param emptyLabel {string} the return value if falsey value
+ */
+export function formatColumnValueWith(
+  value: string | number,
+  fn: (input: number) => string,
+  emptyLabel: string = NO_VALUE,
+): string {
+  if (typeof value === "string") return value;
+  return isNaN(value) ? emptyLabel : fn(value);
+}
+
+/**
+ *
+ * @param input string to check for truncation with '...'
+ * @param end position at which to truncate
+ * @returns original or tooltip-wrapped truncated string
+ */
+export function formatTruncateString(input: string, end: number) {
+  const shouldTruncate = input.length >= end;
+  return shouldTruncate ? input.slice(0, end) + "..." : input;
+}
+/**
+ * base < -2 => 2dp, scientific (small decimals)
+ * base < 0 => 3dp (big decimals)
+ * base < 3 => 2dp (ones, tens, hundreds)
+ * base < 6 => 1dp, K (thousands)
+ * base < 9 => 1dp, M (millions)
+ * base < 12 => 1dp, T (trillions)
+ * base < 15 => 1dp, B (billions)
+ * base >= 15 => 0dp, B (billions)
+ * @param input
+ * @returns a formatted number by abbreviation, based on its order of magnitude
+ */
+export function formatAsAbbreviatedNumber(input: number | string) {
+  // type guard for numbers (e.g. datetime strings)
+  if (typeof input !== "number") return input;
+  else {
+    // convert negatives
+    const inputAsPositive = Math.abs(input);
+
+    const twoDecimal = 10 ** -2;
+    const thousand = 10 ** 3;
+    const million = 10 ** 6;
+    const billion = 10 ** 9;
+    const trillion = 10 ** 12;
+    const trillionPlus = 10 ** 15;
+
+    const isLargeFractionals = inputAsPositive >= twoDecimal;
+    const isOnesTensHundreds = inputAsPositive >= 1;
+    const isThousands = inputAsPositive >= thousand;
+    const isMillions = inputAsPositive >= million;
+    const isBillions = inputAsPositive >= billion;
+    const isSmallTrillions = inputAsPositive >= trillion;
+    const isLargeTrillions = inputAsPositive >= trillionPlus;
+
+    // format as 'T' and beyond (trillions+)
+    if (isLargeTrillions || isSmallTrillions)
+      return new Intl.NumberFormat("en-US", {
+        style: "unit",
+        unit: "liter", //just a placeholder
+        unitDisplay: "narrow",
+        maximumFractionDigits: isLargeTrillions ? 0 : 2,
+      })
+        .format(input / 1.0e12)
+        .replace("L", "T");
+    // format as 'B', 'M', 'K' (billions to thousands)
+    else if (isBillions || isMillions || isThousands) {
+      const lookup = {
+        base: isBillions ? billion : isMillions ? million : thousand,
+        unit: isBillions ? "B" : isMillions ? "M" : "K",
+      };
+      return new Intl.NumberFormat("en-US", {
+        style: "unit",
+        unit: "liter", //just a placeholder
+        unitDisplay: "narrow",
+        maximumFractionDigits: 1,
+      })
+        .format(input / lookup.base)
+        .replace("L", lookup.unit);
+    }
+    // format as unlabeled (1 to 999)
+    else if (isOnesTensHundreds)
+      return new Intl.NumberFormat("en-US", {
+        maximumFractionDigits: 2,
+      }).format(input);
+    // format as fractionals (< 1)
+    else
+      return new Intl.NumberFormat("en-US", {
+        maximumFractionDigits: isLargeFractionals ? 3 : 2,
+        notation:
+          isLargeFractionals || inputAsPositive === 0
+            ? "standard"
+            : "scientific",
+      }).format(input);
+  }
+}
+
+/**
+ * formats as 'Category' instead of 'category' or 'CATEGORY'
+ */
+export function formatTitleCase(input?: string) {
+  if (!input) return NO_VALUE;
+  const start = input.slice(0, 1).toUpperCase();
+  const rest = input.slice(1).toLowerCase();
+  return `${start}${rest}`;
+}

package/recce-source/js/src/utils/transforms.ts

@@ -0,0 +1,81 @@
+import { DataFrame, RowDataTypes, RowObjectType } from "@/lib/api/types";
+
+export function dataFrameToRowObjects(dataFrame: DataFrame): RowObjectType[] {
+  return dataFrame.data.map((row, index) => ({
+    ...dataFrame.columns.reduce<Record<string, RowDataTypes>>(
+      (obj, column, colIndex) => {
+        obj[column.key] = row[colIndex];
+        return obj;
+      },
+      {},
+    ),
+    __status: undefined,
+    _index: index + 1,
+  }));
+}
+
+export function keyToNumber(key: string): number {
+  // Try to parse as a number first
+  const parsed = Number(key);
+
+  // Check if it's a valid number (not NaN, not Infinity)
+  if (!isNaN(parsed) && isFinite(parsed)) {
+    return parsed;
+  }
+
+  // If not a number, hash the string to a stable numeric value
+  return hashStringToNumber(key);
+}
+
+/**
+ * Hashes a string to a stable numeric value using a simple hash algorithm
+ * Based on the Java String.hashCode() implementation
+ */
+export function hashStringToNumber(str: string): number {
+  let hash = 0;
+
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    hash = (hash << 5) - hash + char;
+    hash = hash & hash; // Convert to 32-bit integer
+  }
+
+  // Return absolute value to ensure positive numbers
+  return Math.abs(hash);
+}
+
+export function getCaseInsensitive<T extends RowObjectType>(
+  obj: T,
+  key: string,
+): T[keyof T] | undefined {
+  const lowerKey = key.toLowerCase();
+
+  // First try lowercase
+  if (lowerKey in obj) {
+    return obj[lowerKey as keyof T];
+  }
+
+  // Fall back to case-insensitive search
+  const foundKey = Object.keys(obj).find((k) => k.toLowerCase() === lowerKey) as
+    | keyof T
+    | undefined;
+
+  return foundKey ? obj[foundKey] : undefined;
+}
+
+// Get the value of an object at a given path, case-insensitively
+export function getValueAtPath<T = RowDataTypes>(
+  obj: Record<string, T | undefined>,
+  path: string,
+): T | undefined {
+  let col = obj[path.toLowerCase()];
+  if (!col) {
+    // try upper-case match
+    col = obj[path.toUpperCase()];
+  }
+  if (!col) {
+    // try fallback with strict casing match
+    col = obj[path];
+  }
+  return col;
+}

package/recce-source/js/tsconfig.json

@@ -0,0 +1,47 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "lib": ["DOM", "DOM.iterable", "ESNext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "react-jsx",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["./src/*"],
+      "@theme/*": ["./theme/*"]
+    },
+    "downlevelIteration": true
+  },
+  "types": [
+    "node",
+    "jest",
+    "react/next",
+    "react-dom/next",
+    "jest",
+    "@testing-library/jest-dom"
+  ],
+  "include": [
+    "next-env.d.ts",
+    "jest.globals.d.ts",
+    "**/*.ts",
+    "**/*.tsx",
+    ".next/types/**/*.ts",
+    "eslint.config.mjs",
+    ".next/dev/types/**/*.ts",
+    "./jest.setup.ts"
+  ],
+  "exclude": ["node_modules"]
+}

package/recce-source/macros/README.md

@@ -0,0 +1,8 @@
+## Adapter Macros
+
+Recce use dbt package `audit-hepler` and `dbt_profiler` for value diff and profile diff. However, it encounters some SQL compatibility issues in some warehouse. The folder is to provide adapter-specific macro by mean of the dbt [macro dispatch](https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch) mechanism.
+
+## How to use
+
+1. Copy `recce_<adapter>.sql` to your dbt project `macros/` folder.
+2. Rerun any `dbt` command (e.g. `dbt run`) to make macros available in the `target/manifest.json`

package/recce-source/macros/recce_athena.sql

@@ -0,0 +1,73 @@
+{% macro athena__compare_column_values(a_query, b_query, primary_key, column_to_compare, emojis, a_relation_name, b_relation_name) -%}
+with a_query as (
+    {{ a_query }}
+),
+
+b_query as (
+    {{ b_query }}
+),
+
+joined as (
+    select
+        coalesce(a_query.{{ primary_key }}, b_query.{{ primary_key }}) as {{ primary_key }},
+        a_query.{{ column_to_compare }} as a_query_value,
+        b_query.{{ column_to_compare }} as b_query_value,
+        case
+            when a_query.{{ column_to_compare }} = b_query.{{ column_to_compare }} then '{% if emojis %}✅: {% endif %}perfect match'
+            when a_query.{{ column_to_compare }} is null and b_query.{{ column_to_compare }} is null then '{% if emojis %}✅: {% endif %}both are null'
+            when a_query.{{ primary_key }} is null then '{% if emojis %}🤷: {% endif %}missing from {{ a_relation_name }}'
+            when b_query.{{ primary_key }} is null then '{% if emojis %}🤷: {% endif %}missing from {{ b_relation_name }}'
+            when a_query.{{ column_to_compare }} is null then '{% if emojis %}🤷: {% endif %}value is null in {{ a_relation_name }} only'
+            when b_query.{{ column_to_compare }} is null then '{% if emojis %}🤷: {% endif %}value is null in {{ b_relation_name }} only'
+            when a_query.{{ column_to_compare }} != b_query.{{ column_to_compare }} then '{% if emojis %}❌: {% endif %}‍values do not match'
+            else 'unknown' -- this should never happen
+        end as match_status,
+        case
+            when a_query.{{ column_to_compare }} = b_query.{{ column_to_compare }} then 0
+            when a_query.{{ column_to_compare }} is null and b_query.{{ column_to_compare }} is null then 1
+            when a_query.{{ primary_key }} is null then 2
+            when b_query.{{ primary_key }} is null then 3
+            when a_query.{{ column_to_compare }} is null then 4
+            when b_query.{{ column_to_compare }} is null then 5
+            when a_query.{{ column_to_compare }} != b_query.{{ column_to_compare }} then 6
+            else 7 -- this should never happen
+        end as match_order
+
+    from a_query
+
+    full outer join b_query on a_query.{{ primary_key }} = b_query.{{ primary_key }}
+),
+
+aggregated as (
+    select
+        '{{ column_to_compare }}' as column_name,
+        match_status,
+        match_order,
+        count(*) as count_records
+    from joined
+
+    group by '{{ column_to_compare }}', match_status, match_order
+)
+
+select
+    column_name,
+    match_status,
+    count_records,
+    round(100.0 * count_records / sum(count_records) over (), 2) as percent_of_total
+
+from aggregated
+
+order by match_order
+
+{% endmacro %}
+
+
+{%- macro athena__measure_median(column_name, data_type, cte_name) -%}
+
+{%- if dbt_profiler.is_numeric_dtype(data_type) and not dbt_profiler.is_struct_dtype(data_type) -%}
+    approx_percentile( {{ adapter.quote(column_name) }}, 0.5)
+{%- else -%}
+    cast(null as {{ dbt.type_numeric() }})
+{%- endif -%}
+
+{%- endmacro -%}

package/recce-source/pyproject.toml

@@ -0,0 +1,109 @@
+[project]
+name = "recce"
+dynamic = ["version"]
+description = "Environment diff tool for dbt"
+readme = "README.md"
+authors = [{name = "InfuseAI Dev Team", email = "dev@infuseai.io"}]
+requires-python = ">=3.9"
+license = {text = "Apache-2.0"}
+
+dependencies = [
+    "boto3",
+    "requests>=2.28.1",
+    "ruamel.yaml>=0.18.6",
+    "click>=7.1",
+    "deepdiff>=7.0,<9.0",
+    "portalocker",
+    "fastapi",
+    "itsdangerous",
+    "uvicorn",
+    "pydantic",
+    "jinja2",
+    "rich>=12.0.0",
+    "sentry-sdk",
+    "watchdog",
+    "websockets",
+    "py-markdown-table",
+    "python-dateutil",
+    "python-multipart",
+    "GitPython",
+    "PyGithub",
+    "sqlglot>=27.29.0,<28.0",
+    "pytz",
+    "packaging",
+]
+
+classifiers = [
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Development Status :: 4 - Beta",
+]
+
+[project.optional-dependencies]
+mcp = ["mcp>=1.0.0; python_version >= '3.10'"]
+dev = [
+    "pytest>=4.6",
+    "pytest-asyncio>=0.23,<1.0",
+    "pytest-flake8",
+    "black>=25.1.0",
+    "isort>=6.0.1",
+    "flake8>=7.2.0",
+    "pre-commit>=4.2.0",
+    "pytest-mypy",
+    "pytest-cov",
+    "twine",
+    "tox",
+    "tox-uv",
+    "pandas",
+    "httpx",
+    "dbt-duckdb",
+]
+
+[project.scripts]
+recce = "recce.cli:cli"
+
+[project.urls]
+"Bug Tracker" = "https://github.com/InfuseAI/recce/issues"
+"Homepage" = "https://github.com/InfuseAI/recce"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "recce/VERSION"
+pattern = "(?P<version>.+)"
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "/recce",
+]
+artifacts = [
+    "/recce/data",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["recce"]
+artifacts = [
+    "/recce/data",
+]
+
+[tool.uv.workspace]
+members = ["recce_cloud"]
+
+[tool.black]
+line-length = 120
+
+[tool.isort]
+profile = "black"
+
+[tool.pytest.ini_options]
+markers = [
+    "asyncio: mark test as an async test (used by pytest-asyncio)",
+]
+asyncio_default_fixture_loop_scope = "function"

package/recce-source/recce/VERSION

@@ -0,0 +1 @@
+1.31.0.dev0

package/recce-source/recce/__init__.py

@@ -0,0 +1,84 @@
+import os
+
+import requests
+from packaging.version import Version
+
+
+def is_ci_env():
+    # List of CI environment variables and their expected values
+    ci_environments = {
+        "CI": "true",  # Generic CI indicator
+        "CIRCLECI": "true",  # CircleCI
+        "GITHUB_ACTIONS": "true",  # GitHub Actions
+        "GITLAB_CI": "true",  # GitLab CI
+        "JENKINS_URL": None,  # Jenkins (just needs to exist)
+        "TRAVIS": "true",  # Travis CI
+        "APPVEYOR": "true",  # AppVeyor
+        "DRONE": "true",  # Drone CI
+        "TEAMCITY_VERSION": None,  # TeamCity
+        "BITBUCKET_COMMIT": None,  # Bitbucket Pipelines
+        "BUILDKITE": "true",  # Buildkite
+        "CODEBUILD_BUILD_ID": None,  # AWS CodeBuild
+        "AZURE_PIPELINES": "true",  # Azure Pipelines
+    }
+
+    for env_var, expected_value in ci_environments.items():
+        env_value = os.environ.get(env_var)
+        if env_value is not None:
+            # If we just need the variable to exist
+            if expected_value is None:
+                return True
+            # If we need to match a specific value (case-insensitive)
+            if env_value.lower() == expected_value.lower():
+                return True
+
+    return False
+
+
+def is_recce_cloud_instance():
+    """Check if running in Recce Cloud instance."""
+    return os.environ.get("RECCE_CLOUD_INSTANCE", "false").lower() == "true"
+
+
+def get_runner():
+    # GitHub Action
+    if os.environ.get("GITHUB_ACTIONS", "false") == "true":
+        return "github actions"
+
+    # GitHub Codespace
+    if os.environ.get("CODESPACES", "false") == "true":
+        return "github codespaces"
+
+    # CircleCI
+    if os.environ.get("CIRCLECI", "false") == "true":
+        return "circleci"
+
+    return None
+
+
+def get_version():
+    version_file = os.path.normpath(os.path.join(os.path.dirname(__file__), "VERSION"))
+    with open(version_file) as fh:
+        version = fh.read().strip()
+        return version
+
+
+def fetch_latest_version():
+    current_version = get_version()
+    if "dev" in current_version:
+        # Skip fetching latest version if it's a dev version
+        return current_version
+
+    try:
+        url = "https://pypi.org/pypi/recce/json"
+        response = requests.get(url, timeout=3)
+        response.raise_for_status()
+        data = response.json()
+        return data["info"]["version"]
+    except Exception:
+        return current_version
+
+
+__version__ = get_version()
+__latest_version__ = fetch_latest_version()
+__is_recce_outdated__ = Version(__version__) < Version(__latest_version__)