@shortlink-org/portolan 0.1.0

package/src/lib/data-model.test.ts

@@ -0,0 +1,245 @@
+// The join between a schema and a model, asserted.
+//
+// Most of what matters here is what does NOT get flagged. A type comparison
+// that shouts about every named domain shape is worse than no comparison at
+// all, so the negative cases below carry as much weight as the positive ones.
+
+import { describe, expect, it } from "vitest";
+import { rawCatalog } from "../test-catalog";
+import { buildIndex, validateCatalog } from "../catalog";
+import type { Catalog } from "../catalog";
+import {
+  columnsOfBlock,
+  dbClass,
+  domainClass,
+  outboxOfService,
+  payloadColumn,
+  readersOfStore,
+  storeColumnCount,
+  storedFields,
+  storesOfService,
+  tablesPersisting,
+  typesDisagree,
+  unmappedFields,
+} from "./data-model";
+
+const catalog = validateCatalog(
+  JSON.parse(JSON.stringify(rawCatalog)) as unknown as Catalog,
+);
+const index = buildIndex(catalog);
+
+/**
+ * The catalog with one table's columns pointing at fields the aggregate no
+ * longer declares - the shape a schema takes after a rename that never
+ * reached the migrations, and the thing the drift warning is about.
+ */
+function drifted(): { catalog: Catalog; index: ReturnType<typeof buildIndex> } {
+  const copy = JSON.parse(JSON.stringify(rawCatalog)) as unknown as Catalog;
+  const orders = (copy.stores ?? [])
+    .find((s) => s.id === "shop.oms.pg")
+    ?.tables.find((t) => t.name === "orders");
+  if (!orders) throw new Error("fixture has no orders table");
+  for (const column of orders.columns) if (column.maps) column.maps = `Order.${column.name}Gone`;
+  const validated = validateCatalog(copy);
+  return { catalog: validated, index: buildIndex(validated) };
+}
+
+describe("type classes", () => {
+  it("reads postgres spellings, parameters and all", () => {
+    expect(dbClass("varchar(64)")).toBe("text");
+    expect(dbClass("TIMESTAMPTZ")).toBe("time");
+    expect(dbClass("timestamp with time zone")).toBe("time");
+    expect(dbClass("bigserial")).toBe("int64");
+    expect(dbClass("jsonb")).toBe("json");
+  });
+
+  it("reads go spellings, pointers and slices and all", () => {
+    expect(domainClass("*string")).toBe("text");
+    expect(domainClass("[]byte")).toBe("bytes");
+    expect(domainClass("time.Time")).toBe("time");
+    expect(domainClass("int32")).toBe("int32");
+  });
+
+  it("reads java and typescript spellings", () => {
+    expect(domainClass("long")).toBe("int64");
+    expect(domainClass("Long")).toBe("int64");
+    expect(domainClass("Integer")).toBe("int32");
+    expect(domainClass("String")).toBe("text");
+    expect(domainClass("BigDecimal")).toBe("decimal");
+    expect(domainClass("Instant")).toBe("time");
+    expect(domainClass("Date")).toBe("time");
+  });
+
+  it("gives a width-less int its own class", () => {
+    expect(domainClass("int")).toBe("int");
+    expect(domainClass("*int")).toBe("int");
+  });
+
+  it("knows nothing about a named domain shape, and says so", () => {
+    expect(domainClass("Money")).toBe("unknown");
+    expect(domainClass("GatewayRef")).toBe("unknown");
+    expect(domainClass("number")).toBe("unknown");
+  });
+});
+
+describe("typesDisagree", () => {
+  it("flags the two disagreements the spec names", () => {
+    expect(typesDisagree("uuid", "string")).toBe(true);
+    expect(typesDisagree("bigint", "int32")).toBe(true);
+  });
+
+  it("accepts the same shape spelled differently", () => {
+    expect(typesDisagree("text", "string")).toBe(false);
+    expect(typesDisagree("timestamptz", "time.Time")).toBe(false);
+    expect(typesDisagree("integer", "int32")).toBe(false);
+    expect(typesDisagree("bigserial", "long")).toBe(false);
+    expect(typesDisagree("timestamptz", "Instant")).toBe(false);
+  });
+
+  it("lets a width-less int sit on any integer column, and nothing else", () => {
+    // Go's `int` is whatever the platform says; Postgres `integer` under it
+    // is a choice, not a disagreement. Only a non-integer column is.
+    expect(typesDisagree("integer", "int")).toBe(false);
+    expect(typesDisagree("bigint", "int")).toBe(false);
+    expect(typesDisagree("smallint", "int")).toBe(false);
+    expect(typesDisagree("uuid", "int")).toBe(true);
+    expect(typesDisagree("text", "int")).toBe(true);
+    expect(typesDisagree("numeric", "int")).toBe(true);
+  });
+
+  it("stays quiet when either side is a shape it does not know", () => {
+    // Half the domain types in a catalog are named shapes; guessing at them
+    // would bury the two real disagreements above.
+    expect(typesDisagree("bigint", "Money")).toBe(false);
+    expect(typesDisagree("some_extension_type", "string")).toBe(false);
+  });
+});
+
+describe("storesOfService", () => {
+  it("lists owned stores first, then the ones only read", () => {
+    const stores = storesOfService(index, "delivery.core");
+    expect(stores.map((s) => [s.store.id, s.access])).toEqual([
+      ["delivery.core.pg", "owns"],
+      ["shop.oms.pg", "reads"],
+    ]);
+  });
+
+  it("does not call a service a reader of its own store", () => {
+    const stores = storesOfService(index, "shop.oms");
+    expect(stores.filter((s) => s.access === "reads")).toEqual([]);
+  });
+
+  it("finds the readers of a store from the other side", () => {
+    expect(
+      readersOfStore(catalog, "shop.oms.pg", "shop.oms").map((s) => s.id),
+    ).toEqual(["shop.pricing", "delivery.core"]);
+  });
+});
+
+describe("tablesPersisting", () => {
+  it("lists every table holding an aggregate, with its store", () => {
+    const tables = tablesPersisting(index, "shop.oms.order");
+    expect(tables.map((t) => t.table.name)).toEqual(["orders", "order_items"]);
+    expect(tables[0]?.store.id).toBe("shop.oms.pg");
+  });
+
+  it("is empty for an aggregate nothing stores", () => {
+    expect(tablesPersisting(index, "shop.pricing.price-list")).toEqual([]);
+  });
+});
+
+describe("outboxOfService", () => {
+  it("finds the outbox in a service's own store", () => {
+    expect(outboxOfService(index, "shop.oms")?.table.name).toBe("outbox");
+  });
+
+  it("is null for a service that publishes without one", () => {
+    expect(outboxOfService(index, "payments.ledger")).toBeNull();
+  });
+
+  it("calls the first json column the payload", () => {
+    const outbox = outboxOfService(index, "shop.oms");
+    if (!outbox) throw new Error("fixture has no outbox");
+    expect(payloadColumn(outbox.table)?.name).toBe("payload");
+  });
+});
+
+describe("storedFields", () => {
+  const stored = storedFields(catalog, index, "shop.oms.order.order");
+
+  it("joins each column to the field it maps to", () => {
+    const status = stored.find((s) => s.owner.column.name === "status");
+    expect(status?.path).toBe("status");
+    expect(status?.field?.type).toBe("Status");
+    expect(status?.mismatch).toBe(false);
+  });
+
+  it("marks the uuid column against a string field", () => {
+    const id = stored.find((s) => s.owner.column.name === "id");
+    expect(id?.mismatch).toBe(true);
+  });
+
+  it("leaves a named shape unmarked", () => {
+    const total = stored.find((s) => s.owner.column.name === "total_minor");
+    expect(total?.field?.type).toBe("Money");
+    expect(total?.mismatch).toBe(false);
+  });
+
+  it("keeps a column whose field no longer exists, with no field", () => {
+    const d = drifted();
+    const stored = storedFields(d.catalog, d.index, "shop.oms.order.order");
+    expect(stored.length).toBeGreaterThan(0);
+    expect(stored.every((s) => s.field === null)).toBe(true);
+  });
+
+  it("resolves a nested path against the field the block declares", () => {
+    // "Order.total.amountMinor" is a fact about Money, and Money is read on
+    // its own page; here it is still Order.total.
+    const line = storedFields(catalog, index, "shop.oms.order.order-line");
+    const price = line.find(
+      (s) => s.owner.column.name === "unit_price_minor",
+    );
+    expect(price?.field?.name).toBe("unitPrice");
+  });
+
+  it("is empty for a block nothing stores", () => {
+    expect(storedFields(catalog, index, "shop.oms.order.risk-decision")).toEqual(
+      [],
+    );
+  });
+});
+
+describe("columnsOfBlock", () => {
+  it("finds the columns pointing into a child block", () => {
+    const columns = columnsOfBlock(index, "shop.oms.order.order-line");
+    expect(columns.map((c) => c.column.name)).toEqual([
+      "line_no",
+      "sku",
+      "quantity",
+      "unit_price_minor",
+    ]);
+  });
+});
+
+describe("counts", () => {
+  it("counts every column in a store", () => {
+    const store = (catalog.stores ?? []).find((s) => s.id === "delivery.core.pg");
+    if (!store) throw new Error("fixture has no delivery store");
+    expect(storeColumnCount(store)).toBe(
+      store.tables.reduce((n, t) => n + t.columns.length, 0),
+    );
+  });
+
+  it("counts the fields of an aggregate no column carries", () => {
+    const order = index.aggregateById.get("shop.oms.order");
+    if (!order) throw new Error("fixture has no order");
+    const d = drifted();
+    const drift = d.index.aggregateById.get("shop.oms.order");
+    if (!drift) throw new Error("drifted fixture has no order");
+    // More once the columns point at names the aggregate no longer declares:
+    // that gap is exactly what the drift warning counts.
+    expect(unmappedFields(d.catalog, d.index, drift)).toBeGreaterThan(
+      unmappedFields(catalog, index, order),
+    );
+  });
+});

package/src/lib/data-model.ts

@@ -0,0 +1,399 @@
+// The persistence axis, derived. Pure over a catalog and its index, like
+// derive.ts and backlinks.ts: no DOM, no router, so every number a Data tab or
+// a Persistence section shows can be asserted in a test.
+//
+// The interesting work here is the join between two vocabularies. A schema
+// speaks in `uuid` and `timestamptz`; a domain model speaks in `string` and
+// `time.Time`. Neither is authoritative and neither is wrong, so nothing is
+// rewritten into the other — the two are put side by side and the places where
+// they disagree are marked.
+
+import type {
+  Aggregate,
+  Catalog,
+  CatalogIndex,
+  Column,
+  ColumnOwner,
+  Field,
+  Service,
+  Store,
+  Table,
+  View,
+} from "../catalog";
+import {
+  aggregateBlocks,
+  blockFields,
+  mapsFieldPath,
+  storeViews,
+} from "../catalog";
+
+// ---------------------------------------------------------------------------
+// Stores and services
+// ---------------------------------------------------------------------------
+
+/**
+ * How a service touches a store. Ownership is the store's own claim, so a
+ * service that lists a store it does not own is reading it — which is a fact
+ * the canvas draws (ghosted) rather than an error.
+ */
+export type StoreAccess = "owns" | "reads";
+
+export interface ServiceStore {
+  store: Store;
+  access: StoreAccess;
+}
+
+/**
+ * Every store a service touches: the ones it owns first, in catalog order,
+ * then the ones it only reads. A store it owns but forgot to list is still
+ * owned — the list is a claim about reading, never about ownership.
+ */
+export function storesOfService(
+  index: CatalogIndex,
+  serviceId: string,
+): ServiceStore[] {
+  const owned = index.storesOwnedBy.get(serviceId) ?? [];
+  const out: ServiceStore[] = owned.map((store) => ({
+    store,
+    access: "owns" as const,
+  }));
+
+  const service = index.serviceById.get(serviceId);
+  for (const storeId of service?.stores ?? []) {
+    const store = index.storeById.get(storeId);
+    if (!store || store.owner === serviceId) continue;
+    out.push({ store, access: "reads" });
+  }
+  return out;
+}
+
+/** Services that list a store they do not own — its readers, in catalog order. */
+export function readersOfStore(
+  catalog: Catalog,
+  storeId: string,
+  ownerId: string,
+): Service[] {
+  const out: Service[] = [];
+  for (const context of catalog.contexts) {
+    for (const service of context.services) {
+      if (service.id === ownerId) continue;
+      if ((service.stores ?? []).includes(storeId)) out.push(service);
+    }
+  }
+  return out;
+}
+
+/**
+ * How many columns a store holds, for the header line above an ER canvas.
+ * A view's columns are counted apart: they are not storage, and adding them to
+ * the total would inflate "how big is this schema" with rows that do not exist.
+ */
+export function storeColumnCount(store: Store): number {
+  return store.tables.reduce((n, t) => n + t.columns.length, 0);
+}
+
+/** How many views a store declares. */
+export function storeViewCount(store: Store): number {
+  return storeViews(store).length;
+}
+
+/**
+ * The views presenting an aggregate, alongside the tables that hold it. A
+ * report a service reads its own aggregate through belongs on that aggregate's
+ * page: it is one of the ways the model is actually queried.
+ */
+export interface PresentedView {
+  view: View;
+  store: Store;
+}
+
+export function viewsPresenting(
+  index: CatalogIndex,
+  aggregateId: string,
+): PresentedView[] {
+  const out: PresentedView[] = [];
+  for (const view of index.viewsByAggregate.get(aggregateId) ?? []) {
+    const held = index.viewById.get(view.id);
+    if (held) out.push({ view, store: held.store });
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Aggregates and blocks
+// ---------------------------------------------------------------------------
+
+export interface PersistedTable {
+  table: Table;
+  store: Store;
+}
+
+/**
+ * The tables that hold an aggregate — those naming it in `persists`, plus the
+ * ones naming one of its blocks. A child table pointing at a value object of
+ * the aggregate is still where that aggregate lives.
+ */
+export function tablesPersisting(
+  index: CatalogIndex,
+  aggregateId: string,
+): PersistedTable[] {
+  const out: PersistedTable[] = [];
+  const seen = new Set<string>();
+  for (const table of index.tablesByAggregate.get(aggregateId) ?? []) {
+    const held = index.tableById.get(table.id);
+    if (!held || seen.has(table.id)) continue;
+    seen.add(table.id);
+    out.push({ table, store: held.store });
+  }
+  return out;
+}
+
+/** Columns whose `maps` lands in a block, in table then column order. */
+export function columnsOfBlock(
+  index: CatalogIndex,
+  blockId: string,
+): ColumnOwner[] {
+  return index.columnsByBlock.get(blockId) ?? [];
+}
+
+/**
+ * The outbox a service publishes through, if it has one. An event's delivery
+ * is a fact about the service that announces it, not about the event, so this
+ * is asked once per publisher rather than once per event.
+ */
+export function outboxOfService(
+  index: CatalogIndex,
+  serviceId: string,
+): PersistedTable | null {
+  for (const store of index.storesOwnedBy.get(serviceId) ?? []) {
+    for (const table of store.tables) {
+      if (table.role === "outbox") return { table, store };
+    }
+  }
+  return null;
+}
+
+/** A table's payload column: the first `jsonb`/`json` it holds, if any. */
+export function payloadColumn(table: Table): Column | null {
+  return table.columns.find((c) => dbClass(c.type) === "json") ?? null;
+}
+
+// ---------------------------------------------------------------------------
+// Types. Two vocabularies, one comparison.
+// ---------------------------------------------------------------------------
+
+/**
+ * The shape a type has, once the spelling is set aside. Widths are kept apart
+ * on purpose: `int32` in the model against `bigint` in the column is exactly
+ * the kind of quiet disagreement this whole comparison exists to surface.
+ *
+ * `int` is the one width that is no width: Go's `int` is whatever the platform
+ * says, Java's is what the schema author was not thinking about. Either agrees
+ * with any integer column - a `bigint` under a Go `int` is a choice, not a bug.
+ */
+export type TypeClass =
+  | "uuid"
+  | "text"
+  | "int"
+  | "int16"
+  | "int32"
+  | "int64"
+  | "float"
+  | "decimal"
+  | "bool"
+  | "time"
+  | "json"
+  | "bytes"
+  | "unknown";
+
+/** Strips `[]`, pointers, parameters: "varchar(64)" and "*string" are still one type. */
+function bare(type: string): string {
+  return type
+    .trim()
+    .replace(/^\*+/, "")
+    .replace(/^\[\]/, "")
+    .replace(/\(.*\)$/, "")
+    .replace(/\s+/g, " ")
+    .toLowerCase();
+}
+
+const DB_CLASSES: Record<string, TypeClass> = {
+  uuid: "uuid",
+  text: "text",
+  varchar: "text",
+  "character varying": "text",
+  character: "text",
+  char: "text",
+  citext: "text",
+  smallint: "int16",
+  int2: "int16",
+  smallserial: "int16",
+  integer: "int32",
+  int: "int32",
+  int4: "int32",
+  serial: "int32",
+  bigint: "int64",
+  int8: "int64",
+  bigserial: "int64",
+  numeric: "decimal",
+  decimal: "decimal",
+  money: "decimal",
+  real: "float",
+  float4: "float",
+  "double precision": "float",
+  float8: "float",
+  boolean: "bool",
+  bool: "bool",
+  timestamptz: "time",
+  "timestamp with time zone": "time",
+  timestamp: "time",
+  "timestamp without time zone": "time",
+  date: "time",
+  time: "time",
+  timetz: "time",
+  jsonb: "json",
+  json: "json",
+  bytea: "bytes",
+};
+
+const DOMAIN_CLASSES: Record<string, TypeClass> = {
+  string: "text",
+  "uuid.uuid": "uuid",
+  uuid: "uuid",
+  int16: "int16",
+  int32: "int32",
+  rune: "int32",
+  int64: "int64",
+  int: "int",
+  uint64: "int64",
+  uint32: "int32",
+  float32: "float",
+  float64: "float",
+  bool: "bool",
+  "time.time": "time",
+  "time.duration": "int64",
+  "decimal.decimal": "decimal",
+  "json.rawmessage": "json",
+  byte: "bytes",
+  // Java. `bare` lowercases, so `Integer` and `Long` land here too.
+  long: "int64",
+  integer: "int32",
+  short: "int16",
+  double: "float",
+  boolean: "bool",
+  bigdecimal: "decimal",
+  instant: "time",
+  offsetdatetime: "time",
+  localdatetime: "time",
+  localdate: "time",
+  // TypeScript. `string`, `boolean` above; `number` is deliberately unknown.
+  date: "time",
+};
+
+const INTEGER_CLASSES: ReadonlySet<TypeClass> = new Set([
+  "int",
+  "int16",
+  "int32",
+  "int64",
+]);
+
+export function dbClass(type: string): TypeClass {
+  return DB_CLASSES[bare(type)] ?? "unknown";
+}
+
+export function domainClass(type: string): TypeClass {
+  return DOMAIN_CLASSES[bare(type)] ?? "unknown";
+}
+
+/**
+ * True when a column and the field it maps to disagree about what they hold.
+ *
+ * A type either side does not recognise is NOT a mismatch. Half the domain
+ * types in a catalog are named shapes — Money, GatewayRef — and calling every
+ * one of them a disagreement with `numeric` would bury the two or three real
+ * ones under a page of amber dots.
+ */
+export function typesDisagree(dbType: string, domainType: string): boolean {
+  const a = dbClass(dbType);
+  const b = domainClass(domainType);
+  if (a === "unknown" || b === "unknown") return false;
+  if (a === b) return false;
+  // A width-less integer fits any integer column.
+  return !(b === "int" && INTEGER_CLASSES.has(a));
+}
+
+// ---------------------------------------------------------------------------
+// The join: a column, the field it maps to, and whether they agree.
+// ---------------------------------------------------------------------------
+
+export interface StoredField {
+  owner: ColumnOwner;
+  /** The `maps` path as written, e.g. "Order.CustomerID". */
+  maps: string;
+  /** The field half of that path. */
+  path: string;
+  /** The domain field, when the block actually declares one by that name. */
+  field: Field | null;
+  /** Set when both types are recognised and they are not the same shape. */
+  mismatch: boolean;
+}
+
+/**
+ * Every column stored against a block, joined to the field it claims to carry.
+ * A `maps` naming a field the block does not declare resolves to `field: null`
+ * — the column is still shown, because a column pointing at a field that is no
+ * longer there is precisely what a reader needs to see.
+ */
+export function storedFields(
+  catalog: Catalog,
+  index: CatalogIndex,
+  blockId: string,
+): StoredField[] {
+  const owner = index.blockById.get(blockId);
+  if (!owner) return [];
+  const fields = blockFields(catalog, owner.block);
+
+  return columnsOfBlock(index, blockId).map((columnOwner) => {
+    const maps = columnOwner.column.maps ?? "";
+    const path = mapsFieldPath(maps);
+    // One level only: "Total.Amount" is matched on "Total", the field the
+    // block itself declares, and the nested shape is read on its own page.
+    const head = path.split(".")[0] ?? "";
+    const field = fields.find((f) => f.name === head) ?? null;
+    return {
+      owner: columnOwner,
+      maps,
+      path,
+      field,
+      mismatch: field
+        ? typesDisagree(columnOwner.column.type, field.type)
+        : false,
+    };
+  });
+}
+
+/**
+ * Fields of an aggregate's blocks that no column maps to. This is the evidence
+ * behind "persistence drift": a table claiming to persist an aggregate whose
+ * fields it does not carry is either mid-migration or mis-labelled, and either
+ * way the claim is worth less than it looks.
+ */
+export function unmappedFields(
+  catalog: Catalog,
+  index: CatalogIndex,
+  aggregate: Aggregate,
+): number {
+  let total = 0;
+  let mapped = 0;
+  for (const { block } of aggregateBlocks(aggregate)) {
+    const fields = blockFields(catalog, block);
+    total += fields.length;
+    const paths = new Set(
+      columnsOfBlock(index, block.id).map(
+        (o) => mapsFieldPath(o.column.maps ?? "").split(".")[0] ?? "",
+      ),
+    );
+    mapped += fields.filter((f) => paths.has(f.name)).length;
+  }
+  return total - mapped;
+}