@datarecce/ui 0.1.30 → 0.1.31

package/recce-source/js/src/components/schema/ColumnNameCell.tsx

@@ -0,0 +1,198 @@
+import Box from "@mui/material/Box";
+import CircularProgress from "@mui/material/CircularProgress";
+import IconButton from "@mui/material/IconButton";
+import ListSubheader from "@mui/material/ListSubheader";
+import Menu from "@mui/material/Menu";
+import MenuItem from "@mui/material/MenuItem";
+import Tooltip from "@mui/material/Tooltip";
+import { MouseEvent, useState } from "react";
+import { VscKebabVertical } from "react-icons/vsc";
+import { NodeData } from "@/lib/api/info";
+import {
+  EXPLORE_ACTION,
+  EXPLORE_SOURCE,
+  trackExploreAction,
+} from "@/lib/api/track";
+import { SchemaDiffRow } from "@/lib/dataGrid/generators/toSchemaDataGrid";
+import { useRecceActionContext } from "@/lib/hooks/RecceActionContext";
+import { useRecceInstanceContext } from "@/lib/hooks/RecceInstanceContext";
+import { supportsHistogramDiff } from "../histogram/HistogramDiffForm";
+import { useLineageViewContext } from "../lineage/LineageViewContext";
+
+export function ColumnNameCell({
+  model,
+  row,
+  singleEnv,
+  cllRunning,
+  showMenu = true,
+}: {
+  model: NodeData;
+  row: SchemaDiffRow;
+  singleEnv?: boolean;
+  cllRunning?: boolean;
+  showMenu?: boolean;
+}) {
+  const lineageViewContext = useLineageViewContext();
+  const { runAction } = useRecceActionContext();
+  const { featureToggles } = useRecceInstanceContext();
+  const { name, baseType, currentType, baseIndex, currentIndex } = row;
+  const columnType = currentType ?? baseType;
+  const [anchorEl, setAnchorEl] = useState<null | HTMLElement>(null);
+  const menuOpen = Boolean(anchorEl);
+
+  const handleMenuClick = (event: MouseEvent<HTMLElement>) => {
+    event.stopPropagation();
+    setAnchorEl(event.currentTarget);
+  };
+
+  const handleMenuClose = () => {
+    setAnchorEl(null);
+  };
+
+  const handleProfileDiff = () => {
+    trackExploreAction({
+      action: EXPLORE_ACTION.PROFILE_DIFF,
+      source: EXPLORE_SOURCE.SCHEMA_COLUMN_MENU,
+      node_count: 1,
+    });
+    runAction(
+      "profile_diff",
+      { model: model.name, columns: [name] },
+      { showForm: false },
+    );
+  };
+
+  const handleHistogramDiff = () => {
+    trackExploreAction({
+      action: EXPLORE_ACTION.HISTOGRAM_DIFF,
+      source: EXPLORE_SOURCE.SCHEMA_COLUMN_MENU,
+      node_count: 1,
+    });
+    runAction(
+      "histogram_diff",
+      { model: model.name, column_name: name, column_type: columnType },
+      { showForm: false },
+    );
+  };
+
+  const handleTopkDiff = () => {
+    trackExploreAction({
+      action: EXPLORE_ACTION.TOP_K_DIFF,
+      source: EXPLORE_SOURCE.SCHEMA_COLUMN_MENU,
+      node_count: 1,
+    });
+    runAction(
+      "top_k_diff",
+      { model: model.name, column_name: name, k: 50 },
+      { showForm: false },
+    );
+  };
+
+  const handleValueDiff = () => {
+    trackExploreAction({
+      action: EXPLORE_ACTION.VALUE_DIFF,
+      source: EXPLORE_SOURCE.SCHEMA_COLUMN_MENU,
+      node_count: 1,
+    });
+    runAction(
+      "value_diff",
+      { model: model.name, columns: [name] },
+      { showForm: true },
+    );
+  };
+
+  const addedOrRemoved = !baseType || !currentType;
+  const isCllDisabled =
+    lineageViewContext === undefined ||
+    (baseIndex !== undefined && currentIndex === undefined);
+
+  return (
+    <Tooltip
+      title="View column lineage"
+      placement="top"
+      disableHoverListener={isCllDisabled}
+    >
+      <Box sx={{ display: "flex", alignItems: "center", gap: "3px" }}>
+        <Box
+          sx={{
+            overflow: "hidden",
+            textOverflow: "ellipsis",
+            whiteSpace: "nowrap",
+          }}
+        >
+          {name}
+        </Box>
+        <Box sx={{ flex: 1 }} />
+        {cllRunning && <CircularProgress size={12} color="inherit" />}
+        {showMenu && !singleEnv && model.resource_type !== "source" && (
+          <>
+            <IconButton
+              aria-label="Column options"
+              className="row-context-menu"
+              size="small"
+              disabled={featureToggles.disableDatabaseQuery}
+              onClick={handleMenuClick}
+            >
+              <VscKebabVertical />
+            </IconButton>
+            <Menu
+              anchorEl={anchorEl}
+              open={menuOpen}
+              onClose={handleMenuClose}
+              slotProps={{
+                list: { sx: { lineHeight: "20px" } },
+              }}
+            >
+              <ListSubheader sx={{ m: 0, p: "4px 12px", lineHeight: "20px" }}>
+                Diff
+              </ListSubheader>
+              <MenuItem
+                onClick={() => {
+                  handleProfileDiff();
+                  handleMenuClose();
+                }}
+                disabled={addedOrRemoved}
+                sx={{ fontSize: "0.85rem" }}
+              >
+                Profile Diff
+              </MenuItem>
+              <MenuItem
+                onClick={() => {
+                  handleHistogramDiff();
+                  handleMenuClose();
+                }}
+                disabled={
+                  addedOrRemoved ||
+                  (columnType ? !supportsHistogramDiff(columnType) : true)
+                }
+                sx={{ fontSize: "0.85rem" }}
+              >
+                Histogram Diff
+              </MenuItem>
+              <MenuItem
+                onClick={() => {
+                  handleTopkDiff();
+                  handleMenuClose();
+                }}
+                disabled={addedOrRemoved}
+                sx={{ fontSize: "0.85rem" }}
+              >
+                Top-k Diff
+              </MenuItem>
+              <MenuItem
+                onClick={() => {
+                  handleValueDiff();
+                  handleMenuClose();
+                }}
+                disabled={addedOrRemoved}
+                sx={{ fontSize: "0.85rem" }}
+              >
+                Value Diff
+              </MenuItem>
+            </Menu>
+          </>
+        )}
+      </Box>
+    </Tooltip>
+  );
+}

package/recce-source/js/src/components/schema/SchemaView.tsx

@@ -0,0 +1,337 @@
+import MuiAlert from "@mui/material/Alert";
+import Box from "@mui/material/Box";
+import type {
+  CellClickedEvent,
+  GridApi,
+  GridReadyEvent,
+  RowClassParams,
+} from "ag-grid-community";
+import {
+  forwardRef,
+  Ref,
+  useCallback,
+  useEffect,
+  useMemo,
+  useState,
+} from "react";
+import { NodeData } from "@/lib/api/info";
+import { trackColumnLevelLineage } from "@/lib/api/track";
+import {
+  createDataGridFromData,
+  SchemaDiffRow,
+  SchemaRow,
+} from "@/lib/dataGrid";
+import { useLineageGraphContext } from "@/lib/hooks/LineageGraphContext";
+import {
+  type DataGridHandle,
+  EmptyRowsRenderer,
+  ScreenshotDataGrid,
+} from "../data-grid/ScreenshotDataGrid";
+import { useLineageViewContext } from "../lineage/LineageViewContext";
+
+interface SchemaViewProps {
+  base?: NodeData;
+  current?: NodeData;
+  enableScreenshot?: boolean;
+  showMenu?: boolean;
+}
+
+function PrivateSingleEnvSchemaView(
+  { current, showMenu = true }: { current?: NodeData; showMenu?: boolean },
+  ref: Ref<DataGridHandle>,
+) {
+  const lineageViewContext = useLineageViewContext();
+  const [gridApi, setGridApi] = useState<GridApi<SchemaRow> | null>(null);
+  const [cllRunningMap, setCllRunningMap] = useState<Map<string, boolean>>(
+    new Map(),
+  );
+  const { columns, rows } = useMemo(() => {
+    return createDataGridFromData(
+      { type: "schema_single", columns: current?.columns },
+      { node: current, cllRunningMap, showMenu },
+    );
+  }, [current, cllRunningMap, showMenu]);
+
+  const { lineageGraph } = useLineageGraphContext();
+  const noCatalogCurrent = !lineageGraph?.catalogMetadata.current;
+  let catalogMissingMessage = undefined;
+  if (noCatalogCurrent) {
+    catalogMissingMessage =
+      "catalog.json not found. Run `recce debug` to troubleshoot.";
+  }
+
+  const noSchemaCurrent = current && current.columns === undefined;
+  let schemaMissingMessage = undefined;
+  if (noSchemaCurrent) {
+    schemaMissingMessage =
+      "catalog.json is outdated. Update catalog.json to get schema information.";
+  }
+
+  const handleViewCll = async (columnName: string) => {
+    trackColumnLevelLineage({ action: "view", source: "schema_column" });
+    setCllRunningMap((prev) => new Map(prev).set(columnName, true));
+    const modelId = current?.id;
+    if (modelId) {
+      await lineageViewContext?.showColumnLevelLineage({
+        node_id: modelId,
+        column: columnName,
+      });
+    }
+    setCllRunningMap((prev) => new Map(prev).set(columnName, false));
+  };
+
+  const getRowId = useCallback(
+    (params: { data: SchemaRow }) => {
+      const modelId = current?.id;
+      return `${modelId}-${params.data.name}`;
+    },
+    [current?.id],
+  );
+
+  const cll = lineageViewContext?.viewOptions.column_level_lineage;
+  const selectedRowId = cll ? `${cll.node_id}-${cll.column}` : null;
+
+  // Update row selection when cll changes
+  useEffect(() => {
+    if (!gridApi) return;
+    gridApi.deselectAll();
+    if (selectedRowId) {
+      const rowNode = gridApi.getRowNode(selectedRowId);
+      if (rowNode) {
+        rowNode.setSelected(true);
+      }
+    }
+  }, [gridApi, selectedRowId]);
+
+  const handleGridReady = useCallback((event: GridReadyEvent<SchemaRow>) => {
+    setGridApi(event.api);
+  }, []);
+
+  const getRowClass = (_params: RowClassParams<SchemaRow>) => {
+    if (lineageViewContext !== undefined) {
+      return "row-normal row-selectable";
+    }
+    return "row-normal";
+  };
+
+  const handleCellClicked = async (event: CellClickedEvent<SchemaRow>) => {
+    // Skip if clicking on the menu button
+    const target = event.event?.target as HTMLElement | undefined;
+    if (target?.closest(".row-context-menu")) {
+      return;
+    }
+    const row = event.data;
+    if (row) {
+      await handleViewCll(row.name);
+    }
+  };
+
+  return (
+    <Box sx={{ display: "flex", flexDirection: "column", height: "100%" }}>
+      {catalogMissingMessage ? (
+        <MuiAlert severity="warning" sx={{ fontSize: "12px", p: 1 }}>
+          {catalogMissingMessage}
+        </MuiAlert>
+      ) : schemaMissingMessage ? (
+        <MuiAlert severity="warning" sx={{ fontSize: "12px", p: 1 }}>
+          {schemaMissingMessage}
+        </MuiAlert>
+      ) : (
+        <></>
+      )}
+
+      {rows.length > 0 && (
+        <ScreenshotDataGrid
+          style={{
+            blockSize: "auto",
+            maxHeight: "100%",
+            overflow: "auto",
+            fontSize: "10pt",
+            borderWidth: 1,
+          }}
+          columns={columns}
+          rows={rows}
+          renderers={{ noRowsFallback: <EmptyRowsRenderer /> }}
+          className="rdg-light"
+          ref={ref}
+          getRowId={getRowId}
+          getRowClass={getRowClass}
+          onCellClicked={handleCellClicked}
+          onGridReady={handleGridReady}
+          rowSelection={{ mode: "singleRow" }}
+        />
+      )}
+    </Box>
+  );
+}
+
+export function PrivateSchemaView(
+  { base, current, showMenu = true }: SchemaViewProps,
+  ref: Ref<DataGridHandle>,
+) {
+  const lineageViewContext = useLineageViewContext();
+  const [gridApi, setGridApi] = useState<GridApi<SchemaDiffRow> | null>(null);
+  const [cllRunningMap, setCllRunningMap] = useState<Map<string, boolean>>(
+    new Map(),
+  );
+  const { columns, rows } = useMemo(() => {
+    const resourceType = current?.resource_type ?? base?.resource_type;
+    const node =
+      resourceType &&
+      ["model", "seed", "snapshot", "source"].includes(resourceType)
+        ? (current ?? base)
+        : undefined;
+
+    return createDataGridFromData(
+      { type: "schema_diff", base: base?.columns, current: current?.columns },
+      { node, cllRunningMap, showMenu },
+    );
+  }, [base, current, cllRunningMap, showMenu]);
+
+  const { lineageGraph } = useLineageGraphContext();
+  const noCatalogBase = !lineageGraph?.catalogMetadata.base;
+  const noCatalogCurrent = !lineageGraph?.catalogMetadata.current;
+  let catalogMissingMessage = undefined;
+  if (noCatalogBase && noCatalogCurrent) {
+    catalogMissingMessage =
+      "catalog.json not found on both environments. Run `recce debug` to troubleshoot.";
+  } else if (noCatalogBase) {
+    catalogMissingMessage =
+      "catalog.json not found on base environment. Run `recce debug` to troubleshoot.";
+  } else if (noCatalogCurrent) {
+    catalogMissingMessage =
+      "catalog.json not found on current environment. Run `recce debug` to troubleshoot.";
+  }
+
+  const noSchemaBase = base && base.columns === undefined;
+  const noSchemaCurrent = current && current.columns === undefined;
+  let schemaMissingMessage = undefined;
+  if (noSchemaBase && noSchemaCurrent) {
+    schemaMissingMessage =
+      "catalog.json is outdated on both environments. Update catalog.json to get schema information.";
+  } else if (noSchemaCurrent) {
+    schemaMissingMessage =
+      "catalog.json is outdated on current environment. Update catalog.json to get schema information.";
+  } else if (noSchemaBase) {
+    schemaMissingMessage =
+      "catalog.json is outdated on base environment. Update catalog.json to get schema information.";
+  }
+
+  const handleViewCll = async (columnName: string) => {
+    trackColumnLevelLineage({ action: "view", source: "schema_column" });
+    setCllRunningMap((prev) => new Map(prev).set(columnName, true));
+    const modelId = current?.id ?? base?.id;
+    if (modelId) {
+      await lineageViewContext?.showColumnLevelLineage({
+        node_id: modelId,
+        column: columnName,
+      });
+    }
+    setCllRunningMap((prev) => new Map(prev).set(columnName, false));
+  };
+
+  const getRowId = useCallback(
+    (params: { data: SchemaDiffRow }) => {
+      const modelId = current?.id ?? base?.id;
+      return `${modelId}-${params.data.name}`;
+    },
+    [current?.id, base?.id],
+  );
+
+  const cll = lineageViewContext?.viewOptions.column_level_lineage;
+  const selectedRowId = cll ? `${cll.node_id}-${cll.column}` : null;
+
+  // Update row selection when cll changes
+  useEffect(() => {
+    if (!gridApi) return;
+    gridApi.deselectAll();
+    if (selectedRowId) {
+      const rowNode = gridApi.getRowNode(selectedRowId);
+      if (rowNode) {
+        rowNode.setSelected(true);
+      }
+    }
+  }, [gridApi, selectedRowId]);
+
+  const handleGridReady = useCallback(
+    (event: GridReadyEvent<SchemaDiffRow>) => {
+      setGridApi(event.api);
+    },
+    [],
+  );
+
+  const getRowClass = (params: RowClassParams<SchemaDiffRow>) => {
+    const row = params.data;
+    if (!row) return "row-normal";
+
+    let className;
+    if (row.baseIndex === undefined) {
+      className = "row-added";
+    } else if (row.currentIndex === undefined) {
+      return "row-removed"; // removed column isn't selectable
+    } else {
+      className = "row-normal";
+    }
+    if (lineageViewContext !== undefined) {
+      className += " row-selectable";
+    }
+    return className;
+  };
+
+  const handleCellClicked = async (event: CellClickedEvent<SchemaDiffRow>) => {
+    // Skip if clicking on the menu button
+    const target = event.event?.target as HTMLElement | undefined;
+    if (target?.closest(".row-context-menu")) {
+      return;
+    }
+    const row = event.data;
+    if (!row) return;
+    // Removed columns aren't clickable
+    if (row.baseIndex !== undefined && row.currentIndex === undefined) {
+      return;
+    }
+    await handleViewCll(row.name);
+  };
+
+  return (
+    <Box sx={{ display: "flex", flexDirection: "column", height: "100%" }}>
+      {catalogMissingMessage ? (
+        <MuiAlert severity="warning" sx={{ fontSize: "12px", p: 1 }}>
+          {catalogMissingMessage}
+        </MuiAlert>
+      ) : schemaMissingMessage ? (
+        <MuiAlert severity="warning" sx={{ fontSize: "12px", p: 1 }}>
+          {schemaMissingMessage}
+        </MuiAlert>
+      ) : (
+        <></>
+      )}
+
+      {rows.length > 0 && (
+        <ScreenshotDataGrid
+          style={{
+            blockSize: "auto",
+            maxHeight: "100%",
+            overflow: "auto",
+            fontSize: "0.8rem",
+            borderWidth: 1,
+          }}
+          columns={columns}
+          rows={rows}
+          rowHeight={35}
+          renderers={{ noRowsFallback: <EmptyRowsRenderer /> }}
+          className="rdg-light no-track-pii-safe"
+          ref={ref}
+          getRowId={getRowId}
+          getRowClass={getRowClass}
+          onCellClicked={handleCellClicked}
+          onGridReady={handleGridReady}
+          rowSelection={{ mode: "singleRow" }}
+        />
+      )}
+    </Box>
+  );
+}
+
+export const SchemaView = forwardRef(PrivateSchemaView);
+export const SingleEnvSchemaView = forwardRef(PrivateSingleEnvSchemaView);

package/recce-source/js/src/components/schema/schemaDiff.ts

@@ -0,0 +1,32 @@
+import { NodeData } from "@/lib/api/info";
+
+export function isSchemaChanged(
+  baseSchema: NodeData["columns"],
+  currSchema: NodeData["columns"],
+) {
+  if (!baseSchema || !currSchema) {
+    return undefined;
+  }
+  const baseKeys = Object.keys(baseSchema);
+  const currKeys = Object.keys(currSchema);
+
+  // added, removed
+  if (baseKeys.length !== currKeys.length) {
+    return true;
+  }
+
+  // reordered
+  for (let i = 0; i < baseKeys.length; i++) {
+    if (baseKeys[i] !== currKeys[i]) {
+      return true;
+    }
+  }
+
+  // modified
+  for (const key of currKeys) {
+    if (!baseSchema[key] || baseSchema[key].type !== currSchema[key]?.type) {
+      return true;
+    }
+  }
+  return false;
+}

package/recce-source/js/src/components/schema/style.css

@@ -0,0 +1,134 @@
+/**
+ * Schema diff color palette
+ * Uses CSS custom properties for theme-aware colors
+ */
+
+/* Light mode (default) */
+:root,
+.light {
+  --schema-color-added: rgb(204 255 216);
+  --schema-color-added-hover: rgb(193 246 205);
+  --schema-color-added-active: rgb(166 242 184);
+  --schema-color-removed: rgb(255 215 213);
+  --schema-color-removed-hover: rgb(246 195 193);
+  --schema-color-removed-active: rgb(242 170 166);
+  --schema-color-normal: #ffffff;
+  --schema-color-normal-hover: rgb(241 241 241);
+  --schema-color-normal-active: rgb(230 230 230);
+  --schema-color-reordered: rgb(255 173 21);
+  --schema-color-reordered-hover: rgb(249 164 6);
+  --schema-color-reordered-active: rgb(224 147 6);
+}
+
+/* Dark mode */
+.dark {
+  --schema-color-added: rgb(26 77 26);
+  --schema-color-added-hover: rgb(34 97 34);
+  --schema-color-added-active: rgb(42 117 42);
+  --schema-color-removed: rgb(92 31 31);
+  --schema-color-removed-hover: rgb(112 41 41);
+  --schema-color-removed-active: rgb(132 51 51);
+  --schema-color-normal: #262626;
+  --schema-color-normal-hover: rgb(64 64 64);
+  --schema-color-normal-active: rgb(82 82 82);
+  --schema-color-reordered: rgb(180 83 9);
+  --schema-color-reordered-hover: rgb(146 64 14);
+  --schema-color-reordered-active: rgb(120 53 15);
+}
+
+/* Row backgrounds must be applied to cells because rows use display:contents */
+.row-added {
+  background-color: var(--schema-color-added);
+}
+
+.row-added:hover,
+.row-added[aria-selected="true"],
+.row-added.ag-row-selected {
+  background-color: var(--schema-color-added-hover);
+}
+
+.row-added[aria-selected="true"]:hover,
+.row-added.ag-row-selected:hover {
+  background-color: var(--schema-color-added-active);
+}
+
+.row-removed {
+  background-color: var(--schema-color-removed);
+}
+
+.row-removed:hover,
+.row-removed[aria-selected="true"],
+.row-removed.ag-row-selected {
+  background-color: var(--schema-color-removed-hover);
+}
+
+.row-removed[aria-selected="true"]:hover,
+.row-removed.ag-row-selected:hover {
+  background-color: var(--schema-color-removed-active);
+}
+
+.row-normal {
+  background-color: var(--schema-color-normal);
+}
+
+.row-normal:hover,
+.row-normal[aria-selected="true"],
+.row-normal.ag-row-selected {
+  background-color: var(--schema-color-normal-hover);
+}
+
+.row-normal[aria-selected="true"]:hover,
+.row-normal.ag-row-selected:hover {
+  background-color: var(--schema-color-normal-active);
+}
+
+.row-selectable {
+  cursor: pointer;
+}
+
+/* for general column */
+
+.schema-column[aria-selected="true"] {
+  outline: none;
+  outline-offset: 0;
+}
+
+.schema-column-index {
+  text-align: right;
+  filter: brightness(0.95);
+}
+
+/* for reorder-index and type-change column */
+
+/* Type change badges for merged type column */
+.type-badge {
+  display: inline-block;
+  padding: 2px 6px;
+  border-radius: 4px;
+  font-size: 0.75rem;
+  line-height: 1.2;
+}
+
+.type-badge-removed {
+  background-color: var(--schema-color-removed);
+  margin-right: 4px;
+}
+
+.type-badge-added {
+  background-color: var(--schema-color-added);
+}
+
+.column-index-reordered {
+  background-color: var(--schema-color-reordered);
+}
+
+.row-normal:hover .column-index-reordered,
+.row-normal[aria-selected="true"] .column-index-reordered,
+.row-normal.ag-row-selected .column-index-reordered {
+  background-color: var(--schema-color-reordered-hover);
+}
+
+.row-normal[aria-selected="true"]:hover .column-index-reordered,
+.row-normal.ag-row-selected:hover .column-index-reordered {
+  background-color: var(--schema-color-reordered-active);
+}