@pvidaal07/heydb 0.0.0-stage → 2.0.0-beta.1

package/lib/knowledge/operations.ts

@@ -0,0 +1,223 @@
+import { randomUUID } from "node:crypto";
+
+import { formatName, parseName } from "../model/names.ts";
+import { type Annotation, type Knowledge, PROJECT_TARGET, type Relationship, type RelationshipSide } from "./types.ts";
+
+// Pure operations on Knowledge: each returns a new value and never mutates its
+// input, so a failed validation can never leave half-applied changes behind,
+// and callers decide when (and whether) to save.
+//
+// Entries are kept sorted, so the file written to disk depends only on its
+// content, never on the order in which notes were added. That keeps git diffs
+// small and lets two people annotating different objects merge line by line.
+
+/** Long enough for a real explanation, short enough to keep the file reviewable. */
+export const MAX_TEXT_LENGTH = 4000;
+
+/**
+ * Where time and ids come from. Production uses the real clock and random
+ * UUIDs; tests pass fixed sequences so every result can be compared exactly.
+ */
+export interface KnowledgeOptions {
+	readonly now?: () => string;
+	readonly newId?: () => string;
+}
+
+const realClock = (): string => new Date().toISOString();
+
+export interface AnnotationInput {
+	readonly target: string;
+	readonly text: string;
+	readonly author: string;
+}
+
+export interface RelationshipInput {
+	readonly from: RelationshipSide;
+	readonly to: RelationshipSide;
+	/** Why the relationship exists or how it was found; may be empty. */
+	readonly note: string;
+	readonly author: string;
+}
+
+export function emptyKnowledge(): Knowledge {
+	return { formatVersion: 1, annotations: [], relationships: [] };
+}
+
+export function addAnnotation(
+	knowledge: Knowledge,
+	input: AnnotationInput,
+	options: KnowledgeOptions = {},
+): { knowledge: Knowledge; annotation: Annotation } {
+	// Stored in canonical form, so '"shop".shop' and 'shop.shop' are the same target.
+	return withAnnotation(knowledge, formatName(parseName(input.target)), input, options);
+}
+
+/** A note about the whole project, for the project file; it has no target to resolve. */
+export function addProjectNote(
+	knowledge: Knowledge,
+	input: Omit<AnnotationInput, "target">,
+	options: KnowledgeOptions = {},
+): { knowledge: Knowledge; annotation: Annotation } {
+	return withAnnotation(knowledge, PROJECT_TARGET, input, options);
+}
+
+function withAnnotation(
+	knowledge: Knowledge,
+	target: string,
+	input: Omit<AnnotationInput, "target">,
+	options: KnowledgeOptions,
+): { knowledge: Knowledge; annotation: Annotation } {
+	const { now = realClock, newId = randomUUID } = options;
+	const text = cleanText(input.text, "annotation text", { allowEmpty: false });
+	const author = cleanAuthor(input.author);
+	const timestamp = now();
+	const annotation: Annotation = { id: newId(), target, text, author, createdAt: timestamp, updatedAt: timestamp };
+	return { knowledge: withEntries(knowledge, [...knowledge.annotations, annotation]), annotation };
+}
+
+/** Replaces the text of a note. Who wrote it and when stay as they were. */
+export function editAnnotation(
+	knowledge: Knowledge,
+	id: string,
+	text: string,
+	options: KnowledgeOptions = {},
+): { knowledge: Knowledge; annotation: Annotation } {
+	const { now = realClock } = options;
+	const existing = findById(knowledge.annotations, id, "annotation");
+	const annotation: Annotation = { ...existing, text: cleanText(text, "annotation text", { allowEmpty: false }), updatedAt: now() };
+	const annotations = knowledge.annotations.map((item) => (item.id === id ? annotation : item));
+	return { knowledge: withEntries(knowledge, annotations), annotation };
+}
+
+export function removeAnnotation(knowledge: Knowledge, id: string): { knowledge: Knowledge; annotation: Annotation } {
+	const annotation = findById(knowledge.annotations, id, "annotation");
+	const annotations = knowledge.annotations.filter((item) => item.id !== id);
+	return { knowledge: withEntries(knowledge, annotations), annotation };
+}
+
+export function addRelationship(
+	knowledge: Knowledge,
+	input: RelationshipInput,
+	options: KnowledgeOptions = {},
+): { knowledge: Knowledge; relationship: Relationship } {
+	const { now = realClock, newId = randomUUID } = options;
+	const from = cleanSide(input.from, "from");
+	const to = cleanSide(input.to, "to");
+	if (from.columns.length !== to.columns.length) {
+		throw new Error(
+			`relationship sides must have the same number of columns: from has ${from.columns.length}, to has ${to.columns.length}`,
+		);
+	}
+	// A table may point to itself (staff.manager_id → staff.id); a column may not.
+	if (sameSide(from, to)) throw new Error("relationship from and to are the same columns");
+	const duplicate = knowledge.relationships.find((item) => sameSide(item.from, from) && sameSide(item.to, to));
+	if (duplicate !== undefined) throw new Error(`relationship already exists (id "${duplicate.id}")`);
+
+	const relationship: Relationship = {
+		id: newId(),
+		from,
+		to,
+		note: cleanText(input.note, "relationship note", { allowEmpty: true }),
+		author: cleanAuthor(input.author),
+		createdAt: now(),
+	};
+	return { knowledge: withEntries(knowledge, undefined, [...knowledge.relationships, relationship]), relationship };
+}
+
+export function removeRelationship(
+	knowledge: Knowledge,
+	id: string,
+): { knowledge: Knowledge; relationship: Relationship } {
+	const relationship = findById(knowledge.relationships, id, "relationship");
+	const relationships = knowledge.relationships.filter((item) => item.id !== id);
+	return { knowledge: withEntries(knowledge, undefined, relationships), relationship };
+}
+
+/**
+ * A copy with every entry in canonical order: annotations by target, creation
+ * time and id; relationships by source table, destination table and id.
+ * Sorting by target groups a table's notes with its columns' notes, so a diff
+ * reads like the schema. Plain code-unit comparison: the same on every machine.
+ */
+export function sortKnowledge(knowledge: Knowledge): Knowledge {
+	return {
+		formatVersion: 1,
+		annotations: [...knowledge.annotations].sort((a, b) =>
+			compareKeys([a.target, a.createdAt, a.id], [b.target, b.createdAt, b.id]),
+		),
+		relationships: [...knowledge.relationships].sort((a, b) =>
+			compareKeys([a.from.table, a.to.table, a.id], [b.from.table, b.to.table, b.id]),
+		),
+	};
+}
+
+function withEntries(
+	knowledge: Knowledge,
+	annotations: readonly Annotation[] = knowledge.annotations,
+	relationships: readonly Relationship[] = knowledge.relationships,
+): Knowledge {
+	return sortKnowledge({ formatVersion: 1, annotations, relationships });
+}
+
+function findById<T extends { readonly id: string }>(items: readonly T[], id: string, kind: string): T {
+	const found = items.find((item) => item.id === id);
+	if (found === undefined) throw new Error(`unknown ${kind} id "${id}"`);
+	return found;
+}
+
+function cleanText(value: string, label: string, { allowEmpty }: { allowEmpty: boolean }): string {
+	const text = value.trim();
+	if (!allowEmpty && text === "") throw new Error(`${label} must not be empty`);
+	// Spreading a string splits it by code point, so an emoji counts as one character.
+	const length = [...text].length;
+	if (length > MAX_TEXT_LENGTH) {
+		throw new Error(`${label} is too long: ${length} characters (max ${MAX_TEXT_LENGTH})`);
+	}
+	return text;
+}
+
+function cleanAuthor(value: string): string {
+	const author = value.trim();
+	if (author === "") throw new Error("author must not be empty");
+	return author;
+}
+
+function cleanSide(side: RelationshipSide, label: "from" | "to"): RelationshipSide {
+	const table = canonicalTable(side.table);
+	if (table === null) {
+		throw new Error(`relationship ${label} must be a table name (database.schema.table): got "${side.table}"`);
+	}
+	if (side.columns.length === 0) throw new Error(`relationship ${label}.columns must not be empty`);
+	side.columns.forEach((column, index) => {
+		if (column === "") throw new Error(`relationship ${label}.columns[${index}] must not be empty`);
+		if (side.columns.indexOf(column) !== index) throw new Error(`relationship ${label}.columns repeats "${column}"`);
+	});
+	if (side.connection === undefined) return { table, columns: [...side.columns] };
+	// Whether the connection exists is checked by the caller, which has the config.
+	const connection = side.connection.trim();
+	if (connection === "") throw new Error(`relationship ${label}.connection must not be empty`);
+	return { connection, table, columns: [...side.columns] };
+}
+
+/**
+ * The canonical form of a table name, or null when the name is valid but
+ * names a database or a column instead. Invalid names throw parseName's error.
+ */
+export function canonicalTable(name: string): string | null {
+	const ref = parseName(name);
+	if (ref.table === undefined || ref.column !== undefined) return null;
+	return formatName(ref);
+}
+
+function sameSide(a: RelationshipSide, b: RelationshipSide): boolean {
+	return a.connection === b.connection && a.table === b.table && a.columns.length === b.columns.length && a.columns.every((column, index) => column === b.columns[index]);
+}
+
+function compareKeys(a: readonly string[], b: readonly string[]): number {
+	for (let index = 0; index < a.length; index++) {
+		const left = a[index] ?? "";
+		const right = b[index] ?? "";
+		if (left !== right) return left < right ? -1 : 1;
+	}
+	return 0;
+}

package/lib/knowledge/query.ts

@@ -0,0 +1,129 @@
+import { formatName, parseName } from "../model/names.ts";
+import type { Snapshot } from "../model/snapshot.ts";
+import { canonicalTable } from "./operations.ts";
+import type { Annotation, Knowledge, Relationship, RelationshipSide } from "./types.ts";
+
+// Pure reads over Knowledge. Nothing here changes or deletes an entry.
+
+/** Everything known about one table, at every level that applies to it. */
+export interface TableKnowledge {
+	/** Notes on the table's database (target "db.schema"). */
+	readonly database: readonly Annotation[];
+	readonly table: readonly Annotation[];
+	/** Notes per column name; only columns that have notes appear. */
+	readonly columns: Readonly<Record<string, readonly Annotation[]>>;
+	/** Implicit foreign keys where the table is on either side. */
+	readonly relationships: readonly Relationship[];
+}
+
+/** A relationship read from another connection's knowledge file. */
+export interface ExternalRelationship {
+	/** The connection whose file stores it; its sides without a connection are tables of that one. */
+	readonly connection: string;
+	readonly relationship: Relationship;
+}
+
+/**
+ * Returns database, table and column notes together. heydb v1 only returned
+ * table notes, so an agent asked to annotate columns could never read them back.
+ */
+export function knowledgeForTable(knowledge: Knowledge, tableName: string): TableKnowledge {
+	const table = canonicalTable(tableName);
+	if (table === null) throw new Error(`expected a table name (database.schema.table): got "${tableName}"`);
+	const ref = parseName(table);
+
+	const database: Annotation[] = [];
+	const tableNotes: Annotation[] = [];
+	// A Map, not a plain object: a legacy column called "__proto__" must still be a normal key.
+	const columns = new Map<string, Annotation[]>();
+	for (const annotation of knowledge.annotations) {
+		// Compared segment by segment, never by prefix: "shop.shop.orders" is
+		// not a prefix match for "shop.shop.orders_archive".
+		const target = parseName(annotation.target);
+		if (target.database !== ref.database || target.schema !== ref.schema) continue;
+		if (target.table === undefined) database.push(annotation);
+		else if (target.table !== ref.table) continue;
+		else if (target.column === undefined) tableNotes.push(annotation);
+		else columns.set(target.column, [...(columns.get(target.column) ?? []), annotation]);
+	}
+
+	return {
+		database,
+		table: tableNotes,
+		columns: Object.fromEntries(columns),
+		// A side in another connection is never a table of this one, even with the same name.
+		relationships: knowledge.relationships.filter((item) => isHere(item.from, table) || isHere(item.to, table)),
+	};
+}
+
+/**
+ * The relationships of a knowledge file that point to `table` of another
+ * connection, `connection`: crm's file stores crm.clientes(cod_erp) → erp's
+ * cus(cod), and erp's cus must show it.
+ */
+export function relationshipsInto(knowledge: Knowledge, connection: string, table: string): Relationship[] {
+	const into = (side: RelationshipSide) => side.connection === connection && side.table === table;
+	return knowledge.relationships.filter((item) => into(item.from) || into(item.to));
+}
+
+function isHere(side: RelationshipSide, table: string): boolean {
+	return side.connection === undefined && side.table === table;
+}
+
+export interface Orphans {
+	readonly annotations: readonly Annotation[];
+	readonly relationships: readonly Relationship[];
+}
+
+/** Answers "does this canonical name exist?" for one snapshot. */
+export type Lookup = (name: string) => boolean;
+
+/** The lookup of another connection, or null when it is gone or was never synced. */
+export type OtherConnections = (connection: string) => Lookup | null;
+
+/**
+ * Knowledge whose database, table or column is no longer in the snapshot.
+ * Only reported, never deleted: a note on a dropped column may still be the
+ * only explanation of old rows, backups or the code that used it.
+ *
+ * A side in another connection is checked against `others`: a relationship
+ * whose other connection is gone or unsynced is reported too. Without
+ * `others`, those sides are not checked.
+ */
+export function findOrphans(knowledge: Knowledge, snapshot: Snapshot, others?: OtherConnections): Orphans {
+	const exists = snapshotLookup(snapshot);
+	const missing = (side: RelationshipSide): boolean => {
+		if (side.connection === undefined) return missingInSide(side, exists) !== null;
+		if (others === undefined) return false;
+		const lookup = others(side.connection);
+		return lookup === null || missingInSide(side, lookup) !== null;
+	};
+	return {
+		annotations: knowledge.annotations.filter((annotation) => !exists(annotation.target)),
+		relationships: knowledge.relationships.filter((relationship) => missing(relationship.from) || missing(relationship.to)),
+	};
+}
+
+/** Answers "does this canonical name exist in the snapshot?" for databases, tables and columns. */
+export function snapshotLookup(snapshot: Snapshot): Lookup {
+	const names = new Set<string>();
+	for (const database of snapshot.databases) {
+		for (const schema of database.schemas) {
+			const base = { database: database.name, schema: schema.name };
+			names.add(formatName(base));
+			for (const table of schema.tables) {
+				names.add(formatName({ ...base, table: table.name }));
+				for (const column of table.columns) names.add(formatName({ ...base, table: table.name, column: column.name }));
+			}
+		}
+	}
+	// Re-formatting the parsed name accepts any valid spelling of it.
+	return (name) => names.has(formatName(parseName(name)));
+}
+
+/** The table, or else the first column, of one relationship side missing from a snapshot; null when all exist. */
+export function missingInSide(side: RelationshipSide, exists: Lookup): string | null {
+	const ref = parseName(side.table);
+	const names = [side.table, ...side.columns.map((column) => formatName({ ...ref, column }))];
+	return names.find((name) => !exists(name)) ?? null;
+}

package/lib/knowledge/types.ts

@@ -0,0 +1,64 @@
+// Knowledge is what people and agents learn about a database that its
+// structure does not say: what a column really means, which columns join two
+// tables when there is no foreign key. It lives in
+// .heydb/knowledge/<connection>.json and is committed, like the snapshot.
+//
+// Every entry is anchored to a canonical name (database.schema.table.column),
+// never to a bare table name, so "clientes" in two databases never share notes.
+// Timestamps are ISO 8601 strings; ids come from crypto.randomUUID.
+//
+// Notes about the whole project, above any database ("the ERP uses one
+// database per client"), live in .heydb/knowledge/_project.json with the same
+// shape: only annotations, all with the target PROJECT_TARGET. The leading
+// underscore can never be a connection name, which must start with a letter
+// or digit.
+
+/** The target of every note in the project file. Not a canonical name: those have 2 to 4 segments. */
+export const PROJECT_TARGET = "project";
+
+export interface Knowledge {
+	/** Bumped only when the shape of this file changes incompatibly. */
+	readonly formatVersion: 1;
+	readonly annotations: readonly Annotation[];
+	readonly relationships: readonly Relationship[];
+}
+
+export interface Annotation {
+	readonly id: string;
+	/**
+	 * Canonical name of what the note is about: "db.schema" for the whole
+	 * database, "db.schema.table" for a table, "db.schema.table.column" for a
+	 * column; PROJECT_TARGET in the project file.
+	 */
+	readonly target: string;
+	readonly text: string;
+	readonly author: string;
+	readonly createdAt: string;
+	readonly updatedAt: string;
+}
+
+/**
+ * An implicit foreign key: the database does not enforce it, but the data
+ * follows it (typical in legacy schemas and MyISAM tables). Column lists allow
+ * composite keys; both sides have the same length, matched by position.
+ */
+export interface Relationship {
+	readonly id: string;
+	readonly from: RelationshipSide;
+	readonly to: RelationshipSide;
+	readonly note: string;
+	readonly author: string;
+	readonly createdAt: string;
+}
+
+export interface RelationshipSide {
+	/**
+	 * The connection the table belongs to when it is not the one whose file
+	 * stores the relationship (crm.clientes.cod_erp → erp's cus.cod). Absent
+	 * means that connection, so files written before this field stay valid.
+	 */
+	readonly connection?: string;
+	/** Canonical table name: database.schema.table. */
+	readonly table: string;
+	readonly columns: readonly string[];
+}

package/lib/knowledge/validate.ts

@@ -0,0 +1,151 @@
+import { formatName, parseName } from "../model/names.ts";
+import { canonicalTable } from "./operations.ts";
+import { type Annotation, type Knowledge, PROJECT_TARGET, type Relationship, type RelationshipSide } from "./types.ts";
+
+// Knowledge files are committed and edited by people, merges and older
+// heydb versions, so every field is checked once, at the boundary, like
+// snapshots are in lib/model/validate.ts. The result is rebuilt field by field:
+// unknown fields are dropped and names come back in canonical form. Errors
+// name the exact path, e.g. "relationships[0].to.columns must not be empty".
+//
+// This file has its own small reader because the one in lib/model/validate.ts
+// is private to that module; sharing one is a follow-up.
+
+export function validateKnowledge(data: unknown): Knowledge {
+	return validate(data, (target, path) => canonical(target, path, (name) => formatName(parseName(name))));
+}
+
+/**
+ * The project file: the same shape, but only notes about the whole project.
+ * A note on a database there would be found by nobody looking at that
+ * database, and relationships always belong to a connection.
+ */
+export function validateProjectKnowledge(data: unknown): Knowledge {
+	const knowledge = validate(data, (target, path) => {
+		if (target !== PROJECT_TARGET) throw new Error(`${path} must be "${PROJECT_TARGET}" in the project file: got "${target}"`);
+		return target;
+	});
+	if (knowledge.relationships.length > 0) throw new Error("relationships must be empty: the project file holds only project notes");
+	return knowledge;
+}
+
+type TargetReader = (target: string, path: string) => string;
+
+function validate(data: unknown, readTarget: TargetReader): Knowledge {
+	const root = object(data, "knowledge");
+	if (root.formatVersion !== 1) throw new Error("formatVersion must be 1");
+	const annotations = list(root, "annotations", (record, path) => validateAnnotation(record, path, readTarget));
+	const relationships = list(root, "relationships", validateRelationship);
+	unique(annotations, "annotations");
+	unique(relationships, "relationships");
+	return { formatVersion: 1, annotations, relationships };
+}
+
+function validateAnnotation(record: Record<string, unknown>, path: string, readTarget: TargetReader): Annotation {
+	const target = string(record, "target", path);
+	return {
+		id: id(record, path),
+		target: readTarget(target, `${path}.target`),
+		text: string(record, "text", path),
+		author: string(record, "author", path),
+		createdAt: string(record, "createdAt", path),
+		updatedAt: string(record, "updatedAt", path),
+	};
+}
+
+function validateRelationship(record: Record<string, unknown>, path: string): Relationship {
+	const from = side(record, "from", path);
+	const to = side(record, "to", path);
+	if (to.columns.length !== from.columns.length) {
+		throw new Error(`${path}.to.columns must have as many columns as from.columns (${from.columns.length})`);
+	}
+	return {
+		id: id(record, path),
+		from,
+		to,
+		note: string(record, "note", path),
+		author: string(record, "author", path),
+		createdAt: string(record, "createdAt", path),
+	};
+}
+
+function side(parent: Record<string, unknown>, key: "from" | "to", parentPath: string): RelationshipSide {
+	const path = `${parentPath}.${key}`;
+	const record = object(read(parent, key, parentPath), path);
+	const name = string(record, "table", path);
+	const table = canonical(name, `${path}.table`, canonicalTable);
+	if (table === null) {
+		throw new Error(`${path}.table must be a table name (database.schema.table): got "${name}"`);
+	}
+	const columns = read(record, "columns", path);
+	if (!Array.isArray(columns)) throw new Error(`${path}.columns must be a list`);
+	if (columns.length === 0) throw new Error(`${path}.columns must not be empty`);
+	// Optional: files written before relationships could cross connections have none.
+	const connection = record.connection;
+	if (connection !== undefined && (typeof connection !== "string" || connection === "")) {
+		throw new Error(`${path}.connection must be a non-empty string`);
+	}
+	return {
+		...(connection !== undefined && { connection }),
+		table,
+		columns: columns.map((column, index) => {
+			if (typeof column !== "string") throw new Error(`${path}.columns[${index}] must be a string`);
+			return column;
+		}),
+	};
+}
+
+// Prefixes parseName's own message with the path, so it is clear which entry is bad.
+function canonical<T>(name: string, path: string, convert: (name: string) => T): T {
+	try {
+		return convert(name);
+	} catch (error) {
+		throw new Error(`${path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
+	}
+}
+
+function id(record: Record<string, unknown>, path: string): string {
+	const value = string(record, "id", path);
+	if (value === "") throw new Error(`${path}.id must not be empty`);
+	return value;
+}
+
+// Edits and removals find entries by id; a bad merge that duplicates one
+// would make them touch the wrong entry.
+function unique(items: readonly { readonly id: string }[], path: string): void {
+	const seen = new Map<string, number>();
+	items.forEach((item, index) => {
+		const first = seen.get(item.id);
+		if (first !== undefined) throw new Error(`${path}[${index}].id "${item.id}" is already used by ${path}[${first}]`);
+		seen.set(item.id, index);
+	});
+}
+
+function list<T>(record: Record<string, unknown>, key: string, readItem: (item: Record<string, unknown>, path: string) => T): T[] {
+	const value = read(record, key, "");
+	if (!Array.isArray(value)) throw new Error(`${key} must be a list`);
+	return value.map((item, index) => {
+		const path = `${key}[${index}]`;
+		return readItem(object(item, path), path);
+	});
+}
+
+function string(record: Record<string, unknown>, key: string, path: string): string {
+	const value = read(record, key, path);
+	if (typeof value !== "string") throw new Error(`${join(path, key)} must be a string`);
+	return value;
+}
+
+function read(record: Record<string, unknown>, key: string, path: string): unknown {
+	if (!(key in record)) throw new Error(`${join(path, key)} is missing`);
+	return record[key];
+}
+
+function join(path: string, key: string): string {
+	return path === "" ? key : `${path}.${key}`;
+}
+
+function object(value: unknown, path: string): Record<string, unknown> {
+	if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error(`${path} must be an object`);
+	return value as Record<string, unknown>;
+}