@pvidaal07/heydb 0.0.0-stage → 2.0.0-beta.0

package/lib/model/references.ts

@@ -0,0 +1,251 @@
+import type { Knowledge, Relationship } from "../knowledge/types.ts";
+import { formatName, type ObjectRef, parseName, parseSegments } from "./names.ts";
+import type { Snapshot, Table } from "./snapshot.ts";
+
+// Who points to a table. In a legacy schema this is the strongest evidence of
+// what a table means: "gar" has no comment and looks like "garantías", but 500
+// columns elsewhere called ref_gar or id_gar say "Artículo" in their comments.
+//
+// Three kinds of evidence, strongest first:
+// - declared foreign keys, enforced by the engine;
+// - implicit relationships the team documented (knowledge);
+// - name-pattern guesses: a column called ref_gar, id_gar, cod_gar, gar_id…
+//   in another table of the same database. Nothing enforces them, so they are
+//   always shown as guesses.
+//
+// A table never references itself here: its own foreign keys are shown as
+// outgoing relationships, and a self-reference says nothing about meaning.
+//
+// Only relationships between two tables of this snapshot count. One that
+// crosses into another connection names a table of a different database
+// server, even when the name is the same; heydb_table lists those apart.
+
+/** Guesses kept per table; the rest are only counted. */
+export const MAX_NAME_PATTERN_GUESSES = 200;
+
+const SUMMARY_SIZE = 3;
+
+export interface DeclaredReference {
+	/** Canonical name of the referencing table. */
+	readonly table: string;
+	readonly name: string;
+	readonly columns: readonly string[];
+	readonly toColumns: readonly string[];
+	/** Comment of the first referencing column that has one, or "". */
+	readonly comment: string;
+}
+
+export interface ImplicitReference {
+	readonly relationship: Relationship;
+	/** Comment of the first referencing column that has one, or "". */
+	readonly comment: string;
+}
+
+export interface NamePatternReference {
+	/** Canonical name of the table holding the column. */
+	readonly table: string;
+	readonly column: string;
+	readonly comment: string;
+}
+
+export interface CommentCount {
+	/** The comment as first written, on one line. */
+	readonly comment: string;
+	readonly count: number;
+}
+
+export interface IncomingReferences {
+	readonly declared: readonly DeclaredReference[];
+	readonly implicit: readonly ImplicitReference[];
+	/** At most MAX_NAME_PATTERN_GUESSES, commented columns first. */
+	readonly guesses: readonly NamePatternReference[];
+	/** Guesses found beyond the kept ones. */
+	readonly moreGuesses: number;
+	/** The most frequent comments of all referencing columns, guesses included. */
+	readonly summary: readonly CommentCount[];
+}
+
+export function incomingReferences(snapshot: Snapshot, knowledge: Knowledge, tableName: string): IncomingReferences {
+	if (parseSegments(tableName).length !== 3) throw new Error(`expected a table name (database.schema.table): got "${tableName}"`);
+	const target = parseName(tableName) as Required<Omit<ObjectRef, "column">>;
+	const canonical = formatName(target);
+	const tables = tableIndex(snapshot);
+
+	const declared: DeclaredReference[] = [];
+	for (const [name, { table }] of tables) {
+		if (name === canonical) continue;
+		for (const key of table.foreignKeys) {
+			if (formatName(key.references) !== canonical) continue;
+			declared.push({ table: name, name: key.name, columns: key.columns, toColumns: key.references.columns, comment: commentOf(table, key.columns) });
+		}
+	}
+
+	const implicit = local(knowledge)
+		.filter((relationship) => relationship.to.table === canonical && relationship.from.table !== canonical)
+		.map((relationship) => {
+			const from = tables.get(relationship.from.table)?.table;
+			return { relationship, comment: from === undefined ? "" : commentOf(from, relationship.from.columns) };
+		});
+
+	const covered = coveredColumns(snapshot, knowledge);
+	const wanted = target.table.toLowerCase();
+	const guesses: NamePatternReference[] = [];
+	for (const [name, { database, table }] of tables) {
+		if (database !== target.database || name === canonical || table.kind !== "table") continue;
+		for (const column of table.columns) {
+			if (covered.has(columnKey(name, column.name))) continue;
+			if (namePatternTargets(column.name).includes(wanted)) guesses.push({ table: name, column: column.name, comment: column.comment });
+		}
+	}
+	guesses.sort(
+		(a, b) => Number(a.comment.trim() === "") - Number(b.comment.trim() === "") || compareText(a.table, b.table) || compareText(a.column, b.column),
+	);
+
+	const comments = [...declared.map((item) => item.comment), ...implicit.map((item) => item.comment), ...guesses.map((item) => item.comment)];
+	return {
+		declared: declared.sort((a, b) => compareText(a.table, b.table) || compareText(a.name, b.name)),
+		implicit,
+		guesses: guesses.slice(0, MAX_NAME_PATTERN_GUESSES),
+		moreGuesses: Math.max(0, guesses.length - MAX_NAME_PATTERN_GUESSES),
+		summary: summarize(comments),
+	};
+}
+
+/**
+ * How many incoming references every table has (canonical name → count),
+ * counted exactly like incomingReferences, for the whole snapshot in one pass:
+ * the overview ranks 2,000 tables and cannot scan every column once per table.
+ */
+export function incomingCounts(snapshot: Snapshot, knowledge: Knowledge): Map<string, number> {
+	const counts = new Map<string, number>();
+	const add = (name: string) => counts.set(name, (counts.get(name) ?? 0) + 1);
+	const tables = tableIndex(snapshot);
+
+	for (const [name, { table }] of tables) {
+		for (const key of table.foreignKeys) {
+			const to = formatName(key.references);
+			if (to !== name) add(to);
+		}
+	}
+	for (const relationship of local(knowledge)) {
+		if (relationship.to.table !== relationship.from.table) add(relationship.to.table);
+	}
+
+	// Per database: lowercased table name → canonical names (two when only case differs).
+	const byName = new Map<string, Map<string, string[]>>();
+	for (const [name, { database }] of tables) {
+		const names = byName.get(database) ?? new Map<string, string[]>();
+		byName.set(database, names);
+		const lower = (parseName(name).table as string).toLowerCase();
+		names.set(lower, [...(names.get(lower) ?? []), name]);
+	}
+	const covered = coveredColumns(snapshot, knowledge);
+	for (const [name, { database, table }] of tables) {
+		if (table.kind !== "table") continue;
+		const names = byName.get(database);
+		for (const column of table.columns) {
+			if (covered.has(columnKey(name, column.name))) continue;
+			for (const candidate of namePatternTargets(column.name)) {
+				for (const to of names?.get(candidate) ?? []) if (to !== name) add(to);
+			}
+		}
+	}
+	return counts;
+}
+
+/**
+ * The table names a column name may point to, lowercased: what comes before
+ * or after any "_". ref_gar → ref, gar; cod_gar_ant → cod, cod_gar, gar_ant,
+ * ant. So "gar" matches ref_gar, id_gar, cod_gar, gar_id and gar_precio as a
+ * whole token, but never ref_garantia.
+ */
+export function namePatternTargets(columnName: string): string[] {
+	const lower = columnName.toLowerCase();
+	const targets = new Set<string>();
+	for (let index = lower.indexOf("_"); index !== -1; index = lower.indexOf("_", index + 1)) {
+		const before = lower.slice(0, index);
+		const after = lower.slice(index + 1);
+		if (before !== "") targets.add(before);
+		if (after !== "") targets.add(after);
+	}
+	return [...targets];
+}
+
+interface IndexedTable {
+	readonly database: string;
+	readonly table: Table;
+}
+
+function tableIndex(snapshot: Snapshot): Map<string, IndexedTable> {
+	const tables = new Map<string, IndexedTable>();
+	for (const database of snapshot.databases) {
+		for (const schema of database.schemas) {
+			for (const table of schema.tables) {
+				tables.set(formatName({ database: database.name, schema: schema.name, table: table.name }), { database: database.name, table });
+			}
+		}
+	}
+	return tables;
+}
+
+// A column that is part of a foreign key or a documented relationship already
+// says where it points; guessing from its name would only add noise.
+function coveredColumns(snapshot: Snapshot, knowledge: Knowledge): Set<string> {
+	const covered = new Set<string>();
+	for (const [name, { table }] of tableIndex(snapshot)) {
+		for (const key of table.foreignKeys) for (const column of key.columns) covered.add(columnKey(name, column));
+	}
+	for (const { from } of knowledge.relationships) {
+		if (from.connection === undefined) for (const column of from.columns) covered.add(columnKey(from.table, column));
+	}
+	return covered;
+}
+
+/** The relationships whose two sides are tables of this connection. */
+function local(knowledge: Knowledge): Relationship[] {
+	return knowledge.relationships.filter(({ from, to }) => from.connection === undefined && to.connection === undefined);
+}
+
+function columnKey(table: string, column: string): string {
+	return `${table}\u0000${column}`;
+}
+
+function commentOf(table: Table, columns: readonly string[]): string {
+	for (const name of columns) {
+		const comment = table.columns.find((column) => column.name === name)?.comment ?? "";
+		if (comment.trim() !== "") return comment;
+	}
+	return "";
+}
+
+// "Artículo", "articulo." and " ARTÍCULO " are the same answer. Grouped
+// without case, accents, extra spaces or end punctuation; shown as first written.
+function summarize(comments: readonly string[]): CommentCount[] {
+	const groups = new Map<string, { comment: string; count: number }>();
+	for (const comment of comments) {
+		const shown = comment.replace(/\s+/g, " ").trim();
+		const key = fold(shown);
+		if (key === "") continue;
+		const group = groups.get(key);
+		if (group === undefined) groups.set(key, { comment: shown.replace(/[.:;,]+$/, ""), count: 1 });
+		else group.count += 1;
+	}
+	return [...groups.values()]
+		.sort((a, b) => b.count - a.count || compareText(a.comment, b.comment))
+		.slice(0, SUMMARY_SIZE)
+		.map(({ comment, count }) => ({ comment, count }));
+}
+
+// The same folding as normalizeText in lib/search.ts, which model code must not import.
+function fold(text: string): string {
+	return text
+		.normalize("NFD")
+		.replace(/\p{M}/gu, "")
+		.toLowerCase()
+		.replace(/[.:;,]+$/, "");
+}
+
+// Plain code-unit comparison: the same order on every machine and locale.
+function compareText(a: string, b: string): number {
+	return a < b ? -1 : a > b ? 1 : 0;
+}

package/lib/model/snapshot.ts

@@ -0,0 +1,128 @@
+// A Snapshot is an immutable picture of a database server's structure at one
+// moment: which databases, schemas, tables and columns exist and how they are
+// keyed and related. It never contains row data.
+//
+// Conventions used by every type in this file:
+// - `readonly` everywhere: a snapshot is a fact about the past, never edited.
+// - `null` (not an optional field) when a value was checked and is absent, so
+//   "no default" and "we forgot to read the default" never look the same.
+// - Names are kept exactly as the engine reports them (case included).
+
+export type EngineKind = "mysql" | "mariadb" | "postgresql";
+
+export interface Snapshot {
+	/** Bumped only when the shape of this file changes incompatibly. */
+	readonly formatVersion: 1;
+	readonly engine: EngineInfo;
+	/** ISO 8601 timestamp of the capture. */
+	readonly capturedAt: string;
+	readonly databases: readonly Database[];
+	/** What could not be read (missing privileges, unsupported features). */
+	readonly warnings: readonly Warning[];
+}
+
+export interface EngineInfo {
+	readonly kind: EngineKind;
+	/** Version string as reported by the server, e.g. "8.0.36" or "10.11.6-MariaDB". */
+	readonly version: string;
+}
+
+export interface Database {
+	readonly name: string;
+	/** MySQL and MariaDB have exactly one schema per database, named like the database. */
+	readonly schemas: readonly Schema[];
+}
+
+export interface Schema {
+	readonly name: string;
+	readonly tables: readonly Table[];
+	readonly routines: readonly Routine[];
+}
+
+export type TableKind = "table" | "view";
+
+export interface Table {
+	readonly name: string;
+	readonly kind: TableKind;
+	readonly comment: string;
+	/** Storage engine (InnoDB, MyISAM…); null where the concept does not exist (PostgreSQL). */
+	readonly storageEngine: string | null;
+	/** Approximate number of rows reported by the engine statistics; null when unknown. */
+	readonly rowEstimate: number | null;
+	readonly columns: readonly Column[];
+	/** Primary key column names in key order (not table order). Empty when there is none. */
+	readonly primaryKey: readonly string[];
+	readonly indexes: readonly Index[];
+	readonly foreignKeys: readonly ForeignKey[];
+	readonly triggers: readonly Trigger[];
+	/**
+	 * The SELECT of a view as the engine prints it. null for tables, and null for a
+	 * view whose definition the user may not see (a warning says so).
+	 */
+	readonly viewDefinition: string | null;
+}
+
+export interface Column {
+	readonly name: string;
+	/** 1-based position in the table definition. */
+	readonly position: number;
+	/** Full type as the engine prints it, e.g. "bigint unsigned" or "varchar(255)". */
+	readonly type: string;
+	readonly nullable: boolean;
+	/** Default expression as text; null when the column has no default. */
+	readonly default: string | null;
+	readonly comment: string;
+	readonly autoIncrement: boolean;
+	readonly generated: GeneratedColumn | null;
+	readonly collation: string | null;
+}
+
+export interface GeneratedColumn {
+	readonly expression: string;
+	/** true for STORED, false for VIRTUAL. */
+	readonly stored: boolean;
+}
+
+export interface Index {
+	readonly name: string;
+	/** Column names in index order. */
+	readonly columns: readonly string[];
+	readonly unique: boolean;
+	/** BTREE, HASH, FULLTEXT, GIN… as reported by the engine. */
+	readonly method: string;
+}
+
+export type ReferentialAction = "CASCADE" | "SET NULL" | "SET DEFAULT" | "RESTRICT" | "NO ACTION";
+
+export interface ForeignKey {
+	readonly name: string;
+	/** Local columns, in the same order as `references.columns`. */
+	readonly columns: readonly string[];
+	readonly references: {
+		readonly database: string;
+		readonly schema: string;
+		readonly table: string;
+		readonly columns: readonly string[];
+	};
+	readonly onDelete: ReferentialAction;
+	readonly onUpdate: ReferentialAction;
+}
+
+export interface Routine {
+	readonly name: string;
+	readonly kind: "procedure" | "function";
+}
+
+export interface Trigger {
+	readonly name: string;
+	readonly timing: "BEFORE" | "AFTER" | "INSTEAD OF";
+	readonly events: readonly ("INSERT" | "UPDATE" | "DELETE" | "TRUNCATE")[];
+}
+
+export interface Warning {
+	/** Stable machine-readable code, e.g. "routines_hidden". */
+	readonly code: string;
+	readonly message: string;
+	/** Canonical name of the affected object, when there is one. */
+	readonly target?: string;
+}

package/lib/model/stats.ts

@@ -0,0 +1,97 @@
+import type { Knowledge } from "../knowledge/types.ts";
+import { formatName } from "./names.ts";
+import { incomingCounts } from "./references.ts";
+import type { Snapshot, Table } from "./snapshot.ts";
+
+// What a schema looks like at a glance, before reading any table: how much of
+// it is commented (how far comments can be trusted as documentation), how its
+// tables are named, and which tables the rest points to (its core).
+
+const TOP_PREFIXES = 5;
+const TOP_TABLES = 10;
+
+export interface PrefixCount {
+	readonly prefix: string;
+	readonly count: number;
+}
+
+export interface MainTable {
+	/** Canonical name. */
+	readonly name: string;
+	readonly table: Table;
+	/** Declared, implicit and name-pattern references from other tables. */
+	readonly incoming: number;
+}
+
+export interface SchemaStats {
+	readonly database: string;
+	readonly schema: string;
+	/** Tables only; views are counted apart and left out of every other number. */
+	readonly tables: number;
+	readonly views: number;
+	readonly tablesWithComment: number;
+	readonly columns: number;
+	readonly columnsWithComment: number;
+	/** Prefixes shared by two tables or more, most used first. */
+	readonly prefixes: readonly PrefixCount[];
+	/** The most referenced tables, most first; only tables with references. */
+	readonly mainTables: readonly MainTable[];
+}
+
+/** One entry per schema, in snapshot order (one per database in MySQL and MariaDB). */
+export function schemaStats(snapshot: Snapshot, knowledge: Knowledge, counts = incomingCounts(snapshot, knowledge)): SchemaStats[] {
+	return snapshot.databases.flatMap((database) =>
+		database.schemas.map((schema): SchemaStats => {
+			const tables = schema.tables.filter((table) => table.kind === "table");
+			const columns = tables.flatMap((table) => table.columns);
+
+			const prefixes = new Map<string, number>();
+			for (const table of tables) {
+				const prefix = tablePrefix(table.name);
+				if (prefix !== null) prefixes.set(prefix, (prefixes.get(prefix) ?? 0) + 1);
+			}
+
+			const mainTables = tables
+				.map((table) => {
+					const name = formatName({ database: database.name, schema: schema.name, table: table.name });
+					return { name, table, incoming: counts.get(name) ?? 0 };
+				})
+				.filter((item) => item.incoming > 0)
+				.sort((a, b) => b.incoming - a.incoming || compareText(a.name, b.name))
+				.slice(0, TOP_TABLES);
+
+			return {
+				database: database.name,
+				schema: schema.name,
+				tables: tables.length,
+				views: schema.tables.length - tables.length,
+				tablesWithComment: tables.filter((table) => table.comment.trim() !== "").length,
+				columns: columns.length,
+				columnsWithComment: columns.filter((column) => column.comment.trim() !== "").length,
+				prefixes: [...prefixes]
+					.filter(([, count]) => count >= 2)
+					.map(([prefix, count]) => ({ prefix, count }))
+					.sort((a, b) => b.count - a.count || compareText(a.prefix, b.prefix))
+					.slice(0, TOP_PREFIXES),
+				mainTables,
+			};
+		}),
+	);
+}
+
+/**
+ * The family a table name belongs to: the start of the name up to the first
+ * "_" ("sii_fac" → "sii_") or digit ("log2020" → "log0-9"), lowercased. null
+ * when there is no such start ("gcl", "_tmp", "2020_log").
+ */
+export function tablePrefix(name: string): string | null {
+	const match = /^([^_0-9]+)([_0-9])/.exec(name);
+	if (match === null) return null;
+	const start = (match[1] as string).toLowerCase();
+	return match[2] === "_" ? `${start}_` : `${start}0-9`;
+}
+
+// Plain code-unit comparison: the same order on every machine and locale.
+function compareText(a: string, b: string): number {
+	return a < b ? -1 : a > b ? 1 : 0;
+}