@pvidaal07/heydb 0.0.0-stage → 2.0.0-beta.0

package/lib/usecases/code-scan.ts

@@ -0,0 +1,435 @@
+import { lstat, readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { incomingCounts } from "../model/references.ts";
+import { displayName, formatName } from "../model/names.ts";
+import type { Snapshot } from "../model/snapshot.ts";
+import { HEYDB_DIR } from "../store/project.ts";
+
+// Whether the project next to .heydb/ has code that uses this database, and
+// where. After a sync heydb offers to let the agent read that code and note
+// what it makes explicit (enum values, units, joins without a foreign key).
+// Many projects have none (a heydb project made only to explore a database),
+// and then the notes come from conversations instead.
+//
+// This only finds candidates: a table name in database context, that is SQL
+// (`SELECT … FROM gar`) or an ORM declaration (`protected $table = 'gar'`).
+// A name that is only mentioned (a comment, help text, a variable) does not
+// count: tables called "config" or "para" are ordinary words. It never
+// interprets the code; the agent does that, if the person agrees. The scan runs right after a sync, so it is bounded in files,
+// bytes and time: a huge repository gives a partial answer, never a stuck Pi.
+
+export interface ScanLimits {
+	/** Source files read at most. */
+	readonly maxFiles: number;
+	/** Bytes read at most, all files together. */
+	readonly maxBytes: number;
+	/** Bigger files are skipped (generated code, dumps, minified bundles). */
+	readonly maxFileBytes: number;
+	/** Wall time budget, measured with the `now` clock. */
+	readonly maxMillis: number;
+	/** In a huge schema only the most referenced tables are looked for. */
+	readonly maxTables: number;
+}
+
+export const DEFAULT_SCAN_LIMITS: ScanLimits = {
+	maxFiles: 5000,
+	maxBytes: 50 * 1024 * 1024,
+	maxFileBytes: 512 * 1024,
+	// A large legacy codebase (900 Harbour files) needs about 9 s; progress is shown meanwhile.
+	maxMillis: 10_000,
+	maxTables: 300,
+};
+
+/** Tables kept in the result, most used first, and files kept per table. */
+const MAX_MATCHES = 30;
+const MAX_FILES_PER_TABLE = 5;
+/** Two-letter names ("id", "ab") are everywhere in code: they would only add noise. */
+const MIN_NAME_LENGTH = 3;
+
+/** Folders that hold someone else's code, build output, CI configuration or heydb's own files. */
+const SKIPPED_FOLDERS = new Set([
+	"node_modules",
+	"vendor",
+	".git",
+	HEYDB_DIR,
+	"dist",
+	"build",
+	"coverage",
+	".github",
+	".gitlab",
+	".circleci",
+]);
+/** Files that are never database code: lockfiles and CI configuration. Minified bundles (*.min.js) too. */
+const SKIPPED_FILES = new Set([
+	"pnpm-lock.yaml",
+	"package-lock.json",
+	"yarn.lock",
+	"composer.lock",
+	"Gemfile.lock",
+	"poetry.lock",
+	"Cargo.lock",
+	"go.sum",
+	".gitlab-ci.yml",
+]);
+const SOURCE_EXTENSIONS = new Set([
+	"php",
+	"ts",
+	"tsx",
+	"js",
+	"jsx",
+	"py",
+	"java",
+	"cs",
+	"go",
+	"rb",
+	"sql",
+	"kt",
+	"scala",
+	"xml",
+	"yml",
+	"yaml",
+	// Legacy business code, where ERP databases are usually queried: xBase
+	// (Harbour, Clipper, FoxPro), Delphi, Visual Basic, COBOL, ABL and RPG.
+	"prg",
+	"ch",
+	"pas",
+	"dfm",
+	"vb",
+	"bas",
+	"frm",
+	"cls",
+	"cbl",
+	"cob",
+	"p",
+	"w",
+	"rpgle",
+]);
+
+/** The project's files, with paths relative to its root and "/" separators. */
+export interface CodeFiles {
+	/** Entries of a folder ("" is the root). Symbolic links are "other": never followed. */
+	list(dir: string): Promise<readonly { name: string; kind: "file" | "dir" | "other" }[]>;
+	/** Size in bytes of a regular file; null for anything else or a missing file. */
+	size(path: string): Promise<number | null>;
+	read(path: string): Promise<string>;
+}
+
+export interface ScanOptions {
+	/** Replaced in tests; the default reads the real folder. */
+	readonly files?: CodeFiles;
+	/** Milliseconds; replaced in tests. */
+	readonly now?: () => number;
+	readonly limits?: Partial<ScanLimits>;
+}
+
+export interface TableUsage {
+	/** Display name, e.g. "erp.gar"; tables with the same name in several databases are joined with " / ". */
+	readonly table: string;
+	/** The table name as the schema writes it, e.g. "gar". */
+	readonly name: string;
+	/** Files that use it most, at most 5. */
+	readonly files: readonly string[];
+	/** Occurrences in every scanned file. */
+	readonly hits: number;
+}
+
+export interface SchemaUsage {
+	/** Whether any source file was found at all. */
+	readonly hasCode: boolean;
+	readonly filesScanned: number;
+	/** Files that use at least one table. */
+	readonly filesMatched: number;
+	/** Tables used by at least one file; `matches` keeps the 30 most used. */
+	readonly tablesMatched: number;
+	readonly matches: readonly TableUsage[];
+	/** A limit stopped the scan: more code may use these tables. */
+	readonly truncated: boolean;
+}
+
+export async function findSchemaUsage(projectRoot: string, snapshot: Snapshot, options: ScanOptions = {}): Promise<SchemaUsage> {
+	const limits = { ...DEFAULT_SCAN_LIMITS, ...options.limits };
+	const files = options.files ?? realFiles(projectRoot);
+	const now = options.now ?? (() => performance.now());
+	const deadline = now() + limits.maxMillis;
+	const outOfTime = () => now() > deadline;
+
+	const names = tableNames(snapshot, limits.maxTables);
+	const listed = await listSources(files, limits.maxFiles, outOfTime);
+	let truncated = listed.truncated;
+
+	// lowercased table name → file → hits
+	const found = new Map<string, Map<string, number>>();
+	let filesScanned = 0;
+	let filesMatched = 0;
+	let bytes = 0;
+	for (const path of listed.paths) {
+		if (outOfTime()) {
+			truncated = true;
+			break;
+		}
+		const size = await files.size(path);
+		if (size === null || size > limits.maxFileBytes) continue;
+		if (bytes + size > limits.maxBytes) {
+			truncated = true;
+			break;
+		}
+		bytes += size;
+		let text: string;
+		try {
+			text = await files.read(path);
+		} catch {
+			continue; // deleted or unreadable since it was listed: not this scan's business
+		}
+		filesScanned += 1;
+		let matched = false;
+		for (const token of tableReferences(text)) {
+			const key = token.toLowerCase();
+			if (!names.has(key)) continue;
+			const perFile = found.get(key) ?? new Map<string, number>();
+			found.set(key, perFile);
+			perFile.set(path, (perFile.get(path) ?? 0) + 1);
+			matched = true;
+		}
+		if (matched) filesMatched += 1;
+	}
+
+	const matches = [...found].map(([key, perFile]): TableUsage => {
+		const ranked = [...perFile].sort((a, b) => b[1] - a[1] || compareText(a[0], b[0]));
+		const table = names.get(key) as TableName;
+		return {
+			table: table.display,
+			name: table.name,
+			files: ranked.slice(0, MAX_FILES_PER_TABLE).map(([path]) => path),
+			hits: ranked.reduce((sum, [, hits]) => sum + hits, 0),
+		};
+	});
+	matches.sort((a, b) => b.hits - a.hits || compareText(a.table, b.table));
+	return {
+		hasCode: filesScanned > 0,
+		filesScanned,
+		filesMatched,
+		tablesMatched: matches.length,
+		matches: matches.slice(0, MAX_MATCHES),
+		truncated,
+	};
+}
+
+// The patterns of database context. Each captures the table name as `name`:
+// a whole identifier (letters, digits, "_"), so "gar" never matches in
+// ref_gar or garage. Keywords are case-insensitive.
+
+const ID = "[\\p{L}\\p{N}_]+";
+/** A name as SQL writes it: optionally quoted (` " ' [ ]) and qualified (db.table, db.schema.table). */
+const SQL_NAME = `(?:[\`"'\\[]?${ID}[\`"'\\]]?\\.){0,2}[\`"'\\[]?(?<name>${ID})(?![\\p{L}\\p{N}_])`;
+/**
+ * A database prefix built by code in front of the table, as legacy and web
+ * code often write it: "FROM "+cEmp+".gar" (Harbour, JS), "FROM " . $db . ".gar"
+ * (PHP), `FROM ${db}.gar` (template strings). Bounded, so it stays cheap.
+ */
+const BUILT_PREFIX = `(?:["'\`]\\s*(?:\\+|\\.|&|\\|\\|)\\s*[^"'\`\\n]{1,60}?\\s*(?:\\+|\\.|&|\\|\\|)\\s*["'\`]\\.|\\$\\{[^}\\n]{1,60}\\}\\.)`;
+/** A table name after a SQL keyword: plain, quoted or qualified, or behind a built prefix. */
+const SQL_TARGET = `(?:${BUILT_PREFIX})?${SQL_NAME}`;
+/** A quoted string that holds only the name: 'gar', "gar", `gar`. */
+const QUOTED_NAME = `["'\`](?<name>${ID})["'\`]`;
+/** How far back, within one statement (no ";"), the verb that opens it may be. */
+const STATEMENT_CHARS = 2000;
+
+/**
+ * SQL, anywhere in a file (strings included). Each keyword needs the verb of
+ * its statement, so prose like "read it from config" or "update project"
+ * does not count:
+ * - FROM after SELECT or DELETE (DELETE FROM too)
+ * - JOIN after FROM, or after UPDATE when a SET follows
+ * - INTO after INSERT, REPLACE or MERGE
+ * - UPDATE followed by SET
+ * - TABLE after CREATE, ALTER, DROP, TRUNCATE, RENAME or LOCK
+ *
+ * ORM and mapping declarations:
+ * - `table = 'x'`, `$table = 'x'`, `protected $table = "x"`, `__tablename__ = "x"`
+ *   (SQLAlchemy), `db_table = "x"` (Django), `tableName: "x"` / `tableName = "x"`,
+ *   Doctrine XML `<entity … table="x">`
+ * - `->table('x')`, `DB::table('x')`, `.table("x")`, `->from('x')`, `.from("x")`, `knex("x")`
+ * - `@Table(name = "x")`, `@Entity(name = "x")` (JPA), `[Table("x")]` (C#)
+ * - `table: x` on its own line (YAML mappings)
+ */
+/**
+ * A keyword match plus the context it needs. The keyword is searched first and
+ * the context is checked only around each match, in a bounded window cut at
+ * the statement end (";"). A lookbehind such as (?<=SELECT[^;]{0,2000}?) would
+ * be tried at every position of the file: on a real 500 KB Harbour source that
+ * meant minutes per file.
+ */
+interface ReferenceRule {
+	readonly pattern: RegExp;
+	/** Must appear in the statement before the match, at most `before.chars` back. */
+	readonly before?: { readonly verb: RegExp; readonly chars: number };
+	/** Must appear in the statement after the match, at most `after.chars` ahead. */
+	readonly after?: { readonly verb: RegExp; readonly chars: number };
+}
+
+const REFERENCES: readonly ReferenceRule[] = [
+	{ pattern: new RegExp(`\\bFROM\\s+${SQL_TARGET}`, "dgiu"), before: { verb: /\b(?:SELECT|DELETE)\b/iu, chars: STATEMENT_CHARS } },
+	{ pattern: new RegExp(`\\bJOIN\\s+${SQL_TARGET}`, "dgiu"), before: { verb: /\bFROM\b/iu, chars: STATEMENT_CHARS } },
+	{
+		pattern: new RegExp(`\\bJOIN\\s+${SQL_TARGET}`, "dgiu"),
+		before: { verb: /\bUPDATE\b/iu, chars: STATEMENT_CHARS },
+		after: { verb: /\bSET\b/iu, chars: 500 },
+	},
+	{ pattern: new RegExp(`\\bINTO\\s+${SQL_TARGET}`, "dgiu"), before: { verb: /\b(?:INSERT|REPLACE|MERGE)\b/iu, chars: 100 } },
+	{ pattern: new RegExp(`\\bUPDATE\\s+${SQL_TARGET}`, "dgiu"), after: { verb: /\bSET\b/iu, chars: 500 } },
+	{
+		pattern: new RegExp(`\\bTABLES?\\s+(?:IF\\s+(?:NOT\\s+)?EXISTS\\s+)?${SQL_TARGET}`, "dgiu"),
+		before: { verb: /\b(?:CREATE|ALTER|DROP|TRUNCATE|RENAME|LOCK)\b/iu, chars: 40 },
+	},
+	// Any variable or property whose name says "table" or "tabla": $table, db_table,
+	// __tablename__, tableName, cTabla, oQuery:cTabla. `==` comparisons do not count.
+	{
+		pattern: new RegExp(
+			`(?<![\\p{L}\\p{N}_])\\$?[\\p{L}\\p{N}_]*(?:tabla|table)[\\p{L}\\p{N}_]*\\s*(?::=|=(?!=)|:)\\s*${QUOTED_NAME}`,
+			"dgiu",
+		),
+	},
+	{ pattern: new RegExp(`(?:->|::|\\.)\\s*(?:table|from)\\s*\\(\\s*${QUOTED_NAME}`, "dgu") },
+	{ pattern: new RegExp(`\\bknex\\s*\\(\\s*${QUOTED_NAME}`, "dgu") },
+	{ pattern: new RegExp(`(?:@(?:Table|Entity)\\s*\\([^)]*?\\bname\\s*=\\s*|\\[Table\\s*\\(\\s*)${QUOTED_NAME}`, "dgu") },
+	{ pattern: new RegExp(`^[ \\t]*table:[ \\t]*(?<name>${ID})[ \\t]*\\r?$`, "dgmu") },
+];
+
+/** The part of the statement before `index`: at most `chars` back, cut after the last ";". */
+function statementBefore(text: string, index: number, chars: number): string {
+	const window = text.slice(Math.max(0, index - chars), index);
+	return window.slice(window.lastIndexOf(";") + 1);
+}
+
+/** The part of the statement after `index`: at most `chars` ahead, cut at the first ";". */
+function statementAfter(text: string, index: number, chars: number): string {
+	const window = text.slice(index, index + chars);
+	const end = window.indexOf(";");
+	return end === -1 ? window : window.slice(0, end);
+}
+
+/** Every table name the file uses in database context; one per place, even when two patterns see it. */
+function tableReferences(text: string): string[] {
+	const found = new Map<number, string>();
+	for (const rule of REFERENCES) {
+		for (const match of text.matchAll(rule.pattern)) {
+			const start = match.indices?.groups?.name?.[0];
+			const name = match.groups?.name;
+			if (start === undefined || name === undefined) continue;
+			const { before, after } = rule;
+			if (before !== undefined && !before.verb.test(statementBefore(text, match.index, before.chars))) continue;
+			if (after !== undefined && !after.verb.test(statementAfter(text, match.index + match[0].length, after.chars))) continue;
+			found.set(start, name);
+		}
+	}
+	return [...found.values()];
+}
+
+interface TableName {
+	readonly canonical: string;
+	readonly name: string;
+	readonly display: string;
+}
+
+/** Lowercased table name → how to show it, for the tables worth looking for. */
+function tableNames(snapshot: Snapshot, maxTables: number): Map<string, TableName> {
+	const all: { canonical: string; name: string; display: string }[] = [];
+	for (const database of snapshot.databases) {
+		for (const schema of database.schemas) {
+			for (const table of schema.tables) {
+				if (table.name.length < MIN_NAME_LENGTH) continue;
+				const ref = { database: database.name, schema: schema.name, table: table.name };
+				all.push({ canonical: formatName(ref), name: table.name, display: displayName(ref) });
+			}
+		}
+	}
+	let kept = all;
+	if (all.length > maxTables) {
+		// The tables others point to are the core of a legacy schema, and the ones code uses most.
+		const counts = incomingCounts(snapshot, { formatVersion: 1, annotations: [], relationships: [] });
+		const incoming = (item: (typeof all)[number]) => counts.get(item.canonical) ?? 0;
+		kept = [...all].sort((a, b) => incoming(b) - incoming(a) || compareText(a.canonical, b.canonical)).slice(0, maxTables);
+	}
+	const names = new Map<string, TableName>();
+	for (const item of kept) {
+		const key = item.name.toLowerCase();
+		const previous = names.get(key);
+		names.set(key, previous === undefined ? item : { ...previous, display: `${previous.display} / ${item.display}` });
+	}
+	return names;
+}
+
+/**
+ * The source files to read, in a stable order. git knows which files belong
+ * to the project (and skips what it ignores); without a repository, the
+ * folder is walked. Either way the same folders and extensions are skipped,
+ * so nothing under .heydb/, its secrets file included, is ever read.
+ */
+// The folders are walked directly, never listed through git: the code that
+// uses a database is often a reference copy kept in a folder the project's
+// .gitignore excludes (found on a real ERP: source-code/), and git would hide
+// it. Generated and third-party folders are skipped by name, and the walk is
+// bounded by the file and time limits.
+async function listSources(
+	files: CodeFiles,
+	maxFiles: number,
+	outOfTime: () => boolean,
+): Promise<{ paths: string[]; truncated: boolean }> {
+	const paths: string[] = [];
+	const pending = [""];
+	while (pending.length > 0) {
+		if (outOfTime()) return { paths, truncated: true };
+		const dir = pending.shift() as string;
+		let entries: Awaited<ReturnType<CodeFiles["list"]>>;
+		try {
+			entries = await files.list(dir);
+		} catch {
+			continue; // a folder we may not read
+		}
+		for (const entry of [...entries].sort((a, b) => compareText(a.name, b.name))) {
+			const path = dir === "" ? entry.name : `${dir}/${entry.name}`;
+			if (entry.kind === "dir" && !SKIPPED_FOLDERS.has(entry.name)) pending.push(path);
+			if (entry.kind !== "file" || !isSource(path)) continue;
+			if (paths.length === maxFiles) return { paths, truncated: true };
+			paths.push(path);
+		}
+	}
+	return { paths, truncated: false };
+}
+
+function isSource(path: string): boolean {
+	const segments = path.split("/");
+	if (segments.slice(0, -1).some((segment) => SKIPPED_FOLDERS.has(segment))) return false;
+	const name = segments.at(-1) ?? "";
+	if (SKIPPED_FILES.has(name) || name.toLowerCase().endsWith(".min.js")) return false;
+	const dot = name.lastIndexOf(".");
+	return dot > 0 && SOURCE_EXTENSIONS.has(name.slice(dot + 1).toLowerCase());
+}
+
+function realFiles(root: string): CodeFiles {
+	return {
+		async list(dir) {
+			const entries = await readdir(join(root, dir), { withFileTypes: true });
+			return entries.map((entry) => ({
+				name: entry.name,
+				kind: entry.isFile() ? "file" : entry.isDirectory() ? "dir" : "other",
+			}));
+		},
+		async size(path) {
+			try {
+				const stats = await lstat(join(root, path));
+				return stats.isFile() ? stats.size : null;
+			} catch {
+				return null;
+			}
+		},
+		read: (path) => readFile(join(root, path), "utf8"),
+	};
+}
+
+
+// Plain code-unit comparison: the same order on every machine and locale.
+function compareText(a: string, b: string): number {
+	return a < b ? -1 : a > b ? 1 : 0;
+}

package/lib/usecases/init.ts

@@ -0,0 +1,111 @@
+import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { emptyConfig } from "../config/operations.ts";
+import { stableJson, writeFileAtomic } from "../store/files.ts";
+import { assertNoSymlinks } from "../store/guards.ts";
+import { HEYDB_DIR } from "../store/project.ts";
+
+// .heydb/.gitignore keeps the two local-only things out of git, whatever the
+// project's own .gitignore says: the secrets file and the leftovers of an
+// interrupted atomic write (writeFileAtomic names them *.tmp). The .env rule
+// comes last, which is what ensureEnvIgnored checks for.
+const GITIGNORE = [
+	"# Written by heydb. Everything else in .heydb/ is meant to be committed.",
+	"# Leftovers of an interrupted write.",
+	"*.tmp",
+	"# .env holds local secrets for env:HEYDB_NAME password references; never commit it.",
+	".env",
+	"",
+].join("\n");
+
+export interface InitResult {
+	/** Paths created by this call, relative to the project root; empty when everything existed. */
+	readonly created: readonly string[];
+}
+
+/**
+ * Makes `root` a heydb project. Safe to run again: existing files are never
+ * overwritten (a team may have edited the .gitignore or added connections),
+ * so the only effect of a second run is filling in what is missing.
+ */
+export function initProject(root: string): InitResult {
+	const files: [name: string, content: string][] = [
+		[".gitignore", GITIGNORE],
+		["config.json", stableJson(emptyConfig())],
+	];
+	// Every path is checked before anything is written.
+	for (const [name] of files) assertNoSymlinks(root, name);
+
+	const created: string[] = [];
+	if (createDirectory(join(root, HEYDB_DIR))) created.push(`${HEYDB_DIR}/`);
+	for (const [name, content] of files) {
+		if (createFile(join(root, HEYDB_DIR, name), content)) created.push(`${HEYDB_DIR}/${name}`);
+	}
+	return { created };
+}
+
+const ENV_RULE = "# Added by heydb: .env holds local secrets for env:HEYDB_NAME references; never commit it.\n.env\n";
+
+/**
+ * Makes sure git ignores .heydb/.env before heydb writes a secret there. A
+ * team may have replaced the .gitignore heydb wrote, so it is checked every
+ * time, and the usual file is written when there is none. Returns whether the
+ * file changed.
+ *
+ * heydb does not try to read git's patterns (leading spaces count, "!*" or
+ * "!**\/.env" un-ignore it, and so on). It relies on one rule instead: git
+ * applies the last pattern of this file that matches. So the file is left
+ * alone only when its last rule is exactly ".env" or "/.env"; otherwise the
+ * rule is appended, which makes it the last.
+ */
+export function ensureEnvIgnored(root: string): boolean {
+	assertNoSymlinks(root, ".gitignore");
+	const path = join(root, HEYDB_DIR, ".gitignore");
+	let text: string;
+	try {
+		text = readFileSync(path, "utf8");
+	} catch (error) {
+		if (!hasCode(error, "ENOENT")) throw error;
+		return createFile(path, GITIGNORE);
+	}
+	// Only trailing spaces are dropped: git drops them too, but a leading space
+	// is part of the pattern. Blank lines and comments are not rules.
+	const rules = text
+		.split(/\r?\n/)
+		.map((line) => line.replace(/ +$/, ""))
+		.filter((line) => line !== "" && !line.startsWith("#"));
+	const last = rules[rules.length - 1];
+	if (last === ".env" || last === "/.env") return false;
+	const separator = text === "" || text.endsWith("\n") ? "" : "\n";
+	writeFileAtomic(path, `${text}${separator}${ENV_RULE}`);
+	return true;
+}
+
+function createDirectory(path: string): boolean {
+	try {
+		mkdirSync(path);
+		return true;
+	} catch (error) {
+		if (!hasCode(error, "EEXIST")) throw error;
+		if (!statSync(path).isDirectory()) throw new Error(`${HEYDB_DIR} exists but is not a directory`);
+		return false;
+	}
+}
+
+// The "wx" flag creates the file only if it does not exist, in one system
+// call: there is no gap between "check" and "write" in which another process
+// could create it and then be overwritten.
+function createFile(path: string, content: string): boolean {
+	try {
+		writeFileSync(path, content, { flag: "wx" });
+		return true;
+	} catch (error) {
+		if (hasCode(error, "EEXIST")) return false;
+		throw error;
+	}
+}
+
+function hasCode(error: unknown, code: string): boolean {
+	return error instanceof Error && "code" in error && error.code === code;
+}