@shortlink-org/portolan 0.2.4 → 0.4.0

package/src/er/ErCanvas.tsx

@@ -21,7 +21,7 @@ import {
 } from "@xyflow/react";
 import type { Edge } from "@xyflow/react";
 import "@xyflow/react/dist/style.css";
-import { Columns3, Eye, FileCode2, ImageDown, Maximize2, Search, Waypoints } from "lucide-react";
+import { Columns3, Eye, FileCode2, ImageDown, LayoutGrid, ListFilter, Maximize2, Search, Waypoints, Workflow } from "lucide-react";
 import type { Store } from "../catalog";
 import { storeViews } from "../catalog";
 import { index } from "../data";
@@ -29,10 +29,14 @@ import { DiagramSkeleton } from "../components/DiagramSkeleton";
 import { useSelectionStore } from "../selection/store";
 import { TableNodeCard } from "./TableNode";
 import { ViewNodeCard } from "./ViewNode";
+import { GroupNodeCard } from "./GroupNode";
+import type { ErGroupNode } from "./GroupNode";
 import type { ErFlowNode } from "./RelationCard";
 import { EDGE_W, EDGE_W_LIT } from "../graph/theme";
 import { ErMarkers, MARKER_FLOW, MARKER_MANY, MARKER_ONE } from "./markers";
-import { layoutEr } from "./layout";
+import { canGroup, groupsOf, hideGroups, layoutEr } from "./layout";
+import type { ErGroupFrame, ErGrouping } from "./layout";
+import { CTX_SLOTS } from "../lib/context-color";
 import { lineageChain } from "./lineage";
 import type { LineageMaps } from "./lineage";
 import { erSpec, matchingNodes } from "./spec";
@@ -42,10 +46,16 @@ import { saveCanvasImage, viewportOf } from "../lib/export-canvas";
 import type { ImageKind } from "../lib/export-canvas";
 
 /** Stable across renders: React Flow re-mounts every node when this changes. */
-const erNodeTypes = { erTable: TableNodeCard, erView: ViewNodeCard };
+const erNodeTypes = { erTable: TableNodeCard, erView: ViewNodeCard, erGroup: GroupNodeCard };
+
+/** One flow, roots first; or the cards by model group, the groups packed. */
+type Arrangement = "flow" | "groups";
 
 const DIM = 0.25;
 
+/** More hits than this is most of a schema; framing them is framing everything. */
+const SEARCH_FRAME_LIMIT = 30;
+
 /** The catalog's lineage graph, walked on hover. Built once, with the index. */
 const LINEAGE: LineageMaps = {
   from: index.lineageFrom,
@@ -90,14 +100,18 @@ function Canvas({
   const [hoverEdge, setHoverEdge] = useState<string | null>(null);
   const [layout, setLayout] = useState<{
     positions: Record<string, { x: number; y: number }>;
+    groups: ErGroupFrame[];
     ready: boolean;
-  }>({ positions: {}, ready: false });
+  }>({ positions: {}, groups: [], ready: false });
+  // Null until the reader chooses: a schema big enough is grouped, the rest
+  // flow, and the toggle only appears where the choice exists.
+  const [chosen, setChosen] = useState<Arrangement | null>(null);
 
   const select = useSelectionStore((s) => s.select);
   const clear = useSelectionStore((s) => s.clear);
   const selectionId = useSelectionStore((s) => s.selection?.id ?? null);
 
-  const spec: ErSpec = useMemo(
+  const full: ErSpec = useMemo(
     () =>
       erSpec(index, store, {
         mode,
@@ -108,19 +122,53 @@ function Canvas({
       }),
     [store, mode, expanded, ghost, showViews, showLineage],
   );
+  // Groups the reader switched off. Everything below reads the spec with
+  // them gone, so a hidden group is hidden from the search and the layout
+  // too, not only from the eye.
+  const [hidden, setHidden] = useState<ReadonlySet<string>>(new Set());
+  const spec: ErSpec = useMemo(() => hideGroups(full, hidden), [full, hidden]);
+  const nameOf = useCallback(
+    (aggregate: string) => index.aggregateById.get(aggregate)?.name ?? (aggregate.split(".").at(-1) ?? aggregate),
+    [],
+  );
+  const groupList = useMemo(() => groupsOf(full.nodes, nameOf), [full, nameOf]);
+  const onHide = useCallback((id: string, hide: boolean) => {
+    setHidden((prev) => {
+      const next = new Set(prev);
+      if (hide) next.add(id);
+      else next.delete(id);
+      return next;
+    });
+  }, []);
 
   const matched = useMemo(() => matchingNodes(spec, term), [spec, term]);
+  const matchedIds = useMemo(() => [...matched], [matched]);
+  // Which match the reader is standing on: -1 is "all of them", what typing
+  // gives; Enter steps through them one at a time.
+  const [cursor, setCursor] = useState(-1);
+  useEffect(() => setCursor(-1), [term]);
+
+  const groupable = useMemo(() => canGroup(full), [full]);
+  const arrangement: Arrangement = groupable ? (chosen ?? "groups") : "flow";
 
   useEffect(() => {
     let cancelled = false;
     setLayout((prev) => ({ ...prev, ready: false }));
-    void layoutEr(spec).then((result) => {
-      if (!cancelled) setLayout({ positions: result.positions, ready: true });
+    // The packing aims at the box the picture is drawn in; a box not yet on
+    // screen has no shape, and 2:1 is what the service page's canvas is.
+    const box = wrapper.current;
+    const aspectRatio = box && box.clientWidth > 0 && box.clientHeight > 0 ? box.clientWidth / box.clientHeight : 2;
+    void layoutEr(spec, {
+      grouped: arrangement === "groups",
+      aspectRatio,
+      nameOf,
+    }).then((result) => {
+      if (!cancelled) setLayout({ positions: result.positions, groups: result.groups, ready: true });
     });
     return () => {
       cancelled = true;
     };
-  }, [spec]);
+  }, [spec, arrangement, nameOf]);
 
   // Which columns need an anchor, per table. A column with no relationship
   // gets no handle: React Flow measures every handle it is given, and a wide
@@ -198,6 +246,34 @@ function Canvas({
     return () => observer.disconnect();
   }, [flow, layout.ready]);
 
+  // A search moves the camera, not only the lights. One hit is brought up
+  // close; a handful are framed together; more than that is most of the
+  // schema, so the view stays and the dimming does the work. Clearing the box
+  // fits everything again, so the reader is not left zoomed into a corner.
+  const hadTerm = useRef(false);
+  useEffect(() => {
+    if (!layout.ready) return;
+    if (term.trim() === "") {
+      if (hadTerm.current) flow.fitView({ padding: 0.1, duration: 250 });
+      hadTerm.current = false;
+      return;
+    }
+    hadTerm.current = true;
+    const target = cursor >= 0 ? matchedIds.slice(cursor, cursor + 1) : matchedIds;
+    if (target.length === 0 || target.length > SEARCH_FRAME_LIMIT) return;
+    void flow.fitView({
+      nodes: target.map((id) => ({ id })),
+      padding: target.length === 1 ? 0.35 : 0.2,
+      maxZoom: 1.25,
+      duration: 250,
+    });
+  }, [flow, layout.ready, term, matchedIds, cursor]);
+
+  const onJump = useCallback(() => {
+    if (matchedIds.length === 0) return;
+    setCursor((prev) => (prev + 1) % matchedIds.length);
+  }, [matchedIds]);
+
   const onToggle = useCallback((tableId: string) => {
     setExpanded((prev) => {
       const next = new Set(prev);
@@ -219,7 +295,35 @@ function Canvas({
       : null;
   }, [selectionId]);
 
-  const nodes: ErFlowNode[] = useMemo(
+  // The frames go first so that they are drawn under the cards; nothing
+  // about them is interactive but the label, so a click on one reaches the
+  // pane and clears the selection like empty canvas would.
+  const frames: ErGroupNode[] = useMemo(
+    () =>
+      layout.groups.map((group) => ({
+        id: `group:${group.id}`,
+        type: "erGroup" as const,
+        position: { x: group.x, y: group.y },
+        width: group.width,
+        height: group.height,
+        draggable: false,
+        selectable: false,
+        connectable: false,
+        focusable: false,
+        zIndex: -1,
+        data: {
+          name: group.name,
+          aggregate: group.aggregate,
+          count: group.count,
+          // By the group's place in the full list, so that hiding a
+          // neighbour does not recolour the rest.
+          tint: `var(--ctx-${Math.max(0, groupList.findIndex((g) => g.id === group.id)) % CTX_SLOTS})`,
+        },
+      })),
+    [layout.groups, groupList],
+  );
+
+  const cards: ErFlowNode[] = useMemo(
     () =>
       spec.nodes.map((node) => ({
         id: node.id,
@@ -256,6 +360,7 @@ function Canvas({
       onColumnClick,
     ],
   );
+  const nodes: (ErFlowNode | ErGroupNode)[] = useMemo(() => [...frames, ...cards], [frames, cards]);
 
   const edges: Edge[] = useMemo(
     () =>
@@ -315,7 +420,7 @@ function Canvas({
     [spec, layout.positions, litEdges],
   );
 
-  const fitKey = layout.ready ? `fit-${nodes.length}-${mode}` : "pending";
+  const fitKey = layout.ready ? `fit-${nodes.length}-${mode}-${arrangement}` : "pending";
   const views = storeViews(store).length;
 
   return (
@@ -334,6 +439,13 @@ function Canvas({
         mode={mode}
         onMode={setMode}
         hits={matched.size}
+        cursor={cursor}
+        onJump={onJump}
+        arrangement={groupable ? arrangement : null}
+        onArrangement={setChosen}
+        groups={groupable ? groupList : []}
+        hidden={hidden}
+        onHide={onHide}
         views={views}
         showViews={showViews}
         onShowViews={setShowViews}
@@ -348,7 +460,9 @@ function Canvas({
           nodes={nodes}
           edges={edges}
           nodeTypes={erNodeTypes}
-          onNodeClick={(_, node) => select(node.id, "diagram")}
+          onNodeClick={(_, node) => {
+            if (node.type !== "erGroup") select(node.id, "diagram");
+          }}
           onPaneClick={() => clear("diagram")}
           onEdgeMouseEnter={(_, edge) => setHoverEdge(edge.id)}
           onEdgeMouseLeave={() => setHoverEdge(null)}
@@ -377,6 +491,13 @@ function Toolbar({
   mode,
   onMode,
   hits,
+  cursor,
+  onJump,
+  arrangement,
+  onArrangement,
+  groups,
+  hidden,
+  onHide,
   views,
   showViews,
   onShowViews,
@@ -390,6 +511,16 @@ function Toolbar({
   mode: ColumnMode;
   onMode: (value: ColumnMode) => void;
   hits: number;
+  /** The hit the reader stepped to with Enter, -1 for none yet. */
+  cursor: number;
+  onJump: () => void;
+  /** How the cards are laid out; null when the schema is too small for the choice to matter. */
+  arrangement: Arrangement | null;
+  onArrangement: (value: Arrangement) => void;
+  /** The model groups a reader can switch off; empty when the schema is too small to bother. */
+  groups: ErGrouping[];
+  hidden: ReadonlySet<string>;
+  onHide: (id: string, hide: boolean) => void;
   /** How many views this store has; with none, the toggle is not a choice. */
   views: number;
   showViews: boolean;
@@ -428,13 +559,27 @@ function Toolbar({
         <input
           value={term}
           onChange={(e) => onTerm(e.target.value)}
+          onKeyDown={(e) => {
+            // Enter walks the hits, best first; Escape lets go of the search
+            // and the camera with it.
+            if (e.key === "Enter") {
+              e.preventDefault();
+              onJump();
+            } else if (e.key === "Escape") {
+              e.preventDefault();
+              onTerm("");
+            }
+          }}
           placeholder="find a table or column"
           spellCheck={false}
           aria-label="Find a table or column"
+          title="Enter steps through the hits, Escape clears"
           className="mono w-44 bg-transparent outline-none placeholder:text-muted"
         />
         {term ? (
-          <span className="mono tnum shrink-0 text-muted">{hits}</span>
+          <span className="mono tnum shrink-0 text-muted" aria-live="polite">
+            {cursor >= 0 ? `${cursor + 1}/${hits}` : hits}
+          </span>
         ) : null}
       </label>
 
@@ -459,6 +604,67 @@ function Toolbar({
         </button>
       </div>
 
+      {/* Only a schema big enough to come out as a column offers the choice;
+          on a small one the flow is the picture and a frame would be noise. */}
+      {arrangement ? (
+        <div className="seg bg-canvas" role="group" aria-label="Arrangement">
+          <button
+            type="button"
+            onClick={() => onArrangement("flow")}
+            aria-pressed={arrangement === "flow"}
+            className={arrangement === "flow" ? "is-on" : ""}
+            title="One flow, roots on the left"
+          >
+            <Workflow size={11} aria-hidden className="inline" /> flow
+          </button>
+          <button
+            type="button"
+            onClick={() => onArrangement("groups")}
+            aria-pressed={arrangement === "groups"}
+            className={arrangement === "groups" ? "is-on" : ""}
+            title="Tables by the model group they persist, groups packed to the canvas"
+          >
+            <LayoutGrid size={11} aria-hidden className="inline" /> groups
+          </button>
+        </div>
+      ) : null}
+
+      {/* Which groups are on the canvas. A big schema is read one module at a
+          time, and a module is easier to read with its neighbours gone. */}
+      {groups.length > 0 ? (
+        <details className="relative">
+          <summary
+            className="seg mono cursor-pointer list-none bg-canvas px-2 py-1 text-muted hover:bg-surface"
+            title="Show or hide model groups"
+          >
+            <ListFilter size={11} aria-hidden className="inline" /> {groups.length - hidden.size}/{groups.length}
+          </summary>
+          <div className="absolute left-0 top-full z-20 mt-1 max-h-72 w-64 overflow-y-auto rounded-card border border-line bg-surface p-1 shadow-card">
+            <div className="flex gap-2 px-2 py-1">
+              <button type="button" className="mono text-accent hover:underline" onClick={() => groups.forEach((g) => onHide(g.id, false))}>
+                all
+              </button>
+              <button type="button" className="mono text-accent hover:underline" onClick={() => groups.forEach((g) => onHide(g.id, true))}>
+                none
+              </button>
+            </div>
+            {groups.map((group, i) => (
+              <label key={group.id} className="flex cursor-pointer items-center gap-2 rounded-control px-2 py-1 hover:bg-canvas">
+                <input
+                  type="checkbox"
+                  checked={!hidden.has(group.id)}
+                  onChange={(e) => onHide(group.id, !e.target.checked)}
+                  aria-label={`Show ${group.name}`}
+                />
+                <span className="dot" style={{ color: `var(--ctx-${i % CTX_SLOTS})` }} />
+                <span className="mono min-w-0 flex-1 truncate">{group.name}</span>
+                <span className="mono tnum text-muted">{group.nodes.length}</span>
+              </label>
+            ))}
+          </div>
+        </details>
+      ) : null}
+
       {/* Two toggles rather than one: a reader who wants the tables alone and a
           reader who wants the views without the web of lines that joins them
           are asking different questions, and answering both with one switch

package/src/er/GroupNode.tsx

@@ -0,0 +1,57 @@
+// The frame around one model group's tables, when a schema is laid out by
+// group. It is a background, not a card: it takes no clicks, so the pane
+// behind it still clears the selection, and the cards inside it are drawn on
+// top. Only its label is live, and leads to the group's own page.
+//
+// Each frame has a tint of its own, cycled through the same six hues the
+// contexts use, so that two neighbouring groups read as two at a glance and
+// the label is not the only thing telling them apart.
+
+import type { Node, NodeProps } from "@xyflow/react";
+import { Link } from "react-router";
+import { aggregatePath } from "../routes";
+
+export interface GroupNodeData extends Record<string, unknown> {
+  name: string;
+  aggregate: string | null;
+  count: number;
+  /** A CSS colour the frame is washed with. */
+  tint: string;
+}
+
+export type ErGroupNode = Node<GroupNodeData, "erGroup">;
+
+export function GroupNodeCard({ data, width, height }: NodeProps<ErGroupNode>) {
+  const to = data.aggregate ? aggregatePath(data.aggregate) : null;
+  const label = `${data.name} · ${data.count}`;
+  return (
+    <div
+      className="rounded-card border"
+      style={{
+        width,
+        height,
+        background: `color-mix(in srgb, ${data.tint} 9%, transparent)`,
+        borderColor: `color-mix(in srgb, ${data.tint} 45%, transparent)`,
+        pointerEvents: "none",
+      }}
+      data-er-group={data.aggregate ?? "other"}
+    >
+      <div
+        className="mono inline-block rounded-br-card px-2 py-0.5"
+        style={{
+          pointerEvents: "auto",
+          color: data.tint,
+          background: `color-mix(in srgb, ${data.tint} 14%, transparent)`,
+        }}
+      >
+        {to ? (
+          <Link to={to} className="hover:underline" title={`Open ${data.name}`}>
+            {label}
+          </Link>
+        ) : (
+          <span title="tables that persist no model">{label}</span>
+        )}
+      </div>
+    </div>
+  );
+}

package/src/er/StoreHeader.tsx

@@ -31,6 +31,7 @@ export const STORE_KIND_LABEL: Record<StoreKind, string> = {
   mongodb: "MongoDB",
   clickhouse: "ClickHouse",
   s3: "S3",
+  dynamodb: "DynamoDB",
   other: "store",
 };
 

package/src/er/layout.test.ts

@@ -8,7 +8,8 @@ import { describe, expect, it } from "vitest";
 import { buildIndex, validateCatalog } from "../catalog";
 import { pathologicalCatalog, wideCatalog } from "../lib/scenarios";
 import { erSpec } from "./spec";
-import { layoutEr, layoutInput } from "./layout";
+import type { ErNode, ErSpec } from "./spec";
+import { canGroup, groupsOf, hideGroups, layoutEr, layoutInput } from "./layout";
 
 describe("layoutInput", () => {
   const bad = pathologicalCatalog();
@@ -82,7 +83,89 @@ describe("layoutEr", () => {
 
   it("answers for an empty store without asking elk anything", async () => {
     const result = await layoutEr({ nodes: [], edges: [] });
-    expect(result).toEqual({ positions: {}, width: 0, height: 0 });
+    expect(result).toEqual({ positions: {}, width: 0, height: 0, groups: [] });
+  });
+});
+
+describe("grouping a big schema", () => {
+  /** A card with only what grouping and packing read off it. */
+  const card = (id: string, aggregate: string | null): ErNode =>
+    ({ id, aggregate, width: 208, height: 120, name: id.split(".").at(-1) }) as unknown as ErNode;
+  /** `count` tables over `aggregates` groups, plus `loose` that persist nothing. */
+  const schema = (count: number, aggregates: string[], loose = 0): ErSpec => {
+    const nodes: ErNode[] = [];
+    for (let i = 0; i < count; i += 1) {
+      nodes.push(card(`s.t${i}`, aggregates[i % aggregates.length] ?? null));
+    }
+    for (let i = 0; i < loose; i += 1) nodes.push(card(`s.l${i}`, null));
+    return { nodes, edges: [] };
+  };
+
+  it("groups the cards by the model they persist, biggest first, the loose ones last", () => {
+    const spec = schema(7, ["a.big", "a.small", "a.big"], 2);
+    const groups = groupsOf(spec.nodes, (a) => a.split(".").at(-1)?.toUpperCase() ?? a);
+    expect(groups.map((g) => `${g.name}:${g.nodes.length}`)).toEqual(["BIG:5", "SMALL:2", "other:2"]);
+    expect(groups[2]?.aggregate).toBeNull();
+  });
+
+  it("offers grouping only past the size where one flow becomes a column", () => {
+    expect(canGroup(schema(39, ["a", "b", "c"]))).toBe(false);
+    expect(canGroup(schema(60, ["a", "b"]))).toBe(false);
+    expect(canGroup(schema(60, ["a", "b", "c"]))).toBe(true);
+    // Loose tables count as size but not as a group.
+    expect(canGroup(schema(10, ["a", "b", "c"], 40))).toBe(true);
+    expect(canGroup(schema(10, ["a", "b"], 40))).toBe(false);
+  });
+
+  it("packs the groups and keeps every card inside its own frame", async () => {
+    const spec = schema(48, ["a.x", "a.y", "a.z"], 6);
+    // A key inside a group and one across, which the packing ignores.
+    spec.edges.push(
+      { id: "in", kind: "fk", from: "s.t3", to: "s.t0", fromColumn: "x_id", toColumn: "id", onDelete: null } as unknown as ErSpec["edges"][number],
+      { id: "across", kind: "fk", from: "s.t1", to: "s.t0", fromColumn: "x_id", toColumn: "id", onDelete: null } as unknown as ErSpec["edges"][number],
+    );
+    const laid = await layoutEr(spec, { grouped: true, aspectRatio: 2 });
+    expect(laid.groups.map((g) => g.id)).toEqual(["a.x", "a.y", "a.z", "other"]);
+    for (const node of spec.nodes) {
+      const at = laid.positions[node.id];
+      expect(at).toBeDefined();
+      const frame = laid.groups.find((g) => g.id === (node.aggregate ?? "other"));
+      expect(frame).toBeDefined();
+      if (!at || !frame) continue;
+      expect(at.x).toBeGreaterThanOrEqual(frame.x);
+      expect(at.y).toBeGreaterThanOrEqual(frame.y);
+      expect(at.x + node.width).toBeLessThanOrEqual(frame.x + frame.width + 0.5);
+      expect(at.y + node.height).toBeLessThanOrEqual(frame.y + frame.height + 0.5);
+    }
+    // Frames do not overlap.
+    for (const a of laid.groups) {
+      for (const b of laid.groups) {
+        if (a.id === b.id) continue;
+        const apart = a.x + a.width <= b.x || b.x + b.width <= a.x || a.y + a.height <= b.y || b.y + b.height <= a.y;
+        expect(apart).toBe(true);
+      }
+    }
+    // Packed near the box's shape rather than as a column.
+    expect(laid.width / laid.height).toBeGreaterThan(1);
+    // The key inside a group put its parent left of its child.
+    expect(laid.positions["s.t0"]?.x ?? 0).toBeLessThan(laid.positions["s.t3"]?.x ?? 0);
+  });
+
+  it("hides a group with every edge that touched it, and leaves the rest as it was", () => {
+    const spec = schema(6, ["a.x", "a.y"], 2);
+    spec.edges.push(
+      { id: "in-x", kind: "fk", from: "s.t2", to: "s.t0", fromColumn: "x_id", toColumn: "id", onDelete: null } as unknown as ErSpec["edges"][number],
+      { id: "x-y", kind: "fk", from: "s.t1", to: "s.t0", fromColumn: "x_id", toColumn: "id", onDelete: null } as unknown as ErSpec["edges"][number],
+    );
+    expect(hideGroups(spec, new Set())).toBe(spec);
+    const without = hideGroups(spec, new Set(["a.y", "other"]));
+    expect(without.nodes.map((n) => n.id)).toEqual(["s.t0", "s.t2", "s.t4"]);
+    expect(without.edges.map((e) => e.id)).toEqual(["in-x"]);
+  });
+
+  it("stays one flow unless asked", async () => {
+    const laid = await layoutEr(schema(45, ["a", "b", "c"]));
+    expect(laid.groups).toEqual([]);
   });
 });