nilometer 0.0.0-stage → 0.2.0

package/dist/core/db/database.js

@@ -0,0 +1,182 @@
+/**
+ * @file Opening the SQLite database and applying numbered SQL migrations (docs/development.md P3.1).
+ *
+ * Implements README § Tech stack ("analysis logic lives in SQL") and D-093 (Node's built-in `node:sqlite`,
+ * which replaced better-sqlite3 so an npm install needs no native module).
+ * Schema is owned by numbered files in `core/schema/` (`001_name.sql`, `002_name.sql`, ...), never
+ * by application code. Migrations run every time the database is opened, including for read-only
+ * commands, so an older database is always brought up to date before it is queried. phuryn's
+ * tool crashed reading an old schema because only its write path migrated (phuryn#153).
+ */
+import { readFileSync, readdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { ensurePrivateDir, ensurePrivateFile, tightenMode } from "../install/private-files.js";
+/**
+ * Runs `work` in one transaction: committed when it returns, rolled back when it throws. Inside another
+ * transaction it becomes a savepoint, so an inner failure undoes only the inner work, as better-sqlite3's
+ * `db.transaction()` did before D-093.
+ * @param db - An open database.
+ * @param work - The statements to run together.
+ * @returns What `work` returns.
+ * @throws {Error} Whatever `work` throws, after the rollback.
+ */
+export function transaction(db, work) {
+    const nested = db.isTransaction;
+    // A savepoint name only has to be unique among the ones open at once, and nesting is strictly LIFO.
+    const begin = nested ? "SAVEPOINT nilometer_nested" : "BEGIN";
+    db.exec(begin);
+    try {
+        const result = work();
+        db.exec(nested ? "RELEASE nilometer_nested" : "COMMIT");
+        return result;
+    }
+    catch (error) {
+        if (nested) {
+            // ROLLBACK TO keeps the savepoint open, so it's released too, leaving the outer transaction as it was.
+            db.exec("ROLLBACK TO nilometer_nested");
+            db.exec("RELEASE nilometer_nested");
+        }
+        else if (db.isTransaction) {
+            // SQLite may already have rolled back on its own (a full disk, an interrupt); rolling back twice errors.
+            db.exec("ROLLBACK");
+        }
+        throw error;
+    }
+}
+/**
+ * Views a BLOB read from the database as a Buffer, without copying it. node:sqlite returns blobs as
+ * plain `Uint8Array`s, whose `toString("utf8")` doesn't decode: it joins the byte values with commas,
+ * so a line read without this would parse as malformed instead of failing (D-093).
+ * @param blob - A BLOB column's value.
+ * @returns The same bytes as a Buffer.
+ */
+export function blobBytes(blob) {
+    return Buffer.from(blob.buffer, blob.byteOffset, blob.byteLength);
+}
+/**
+ * Reads one pragma's value.
+ * @param db - An open database.
+ * @param name - The pragma, e.g. `journal_mode`.
+ * @returns The first column of its first row, or undefined when it returns none.
+ */
+export function pragmaValue(db, name) {
+    const row = db.prepare(`PRAGMA ${name}`).get();
+    return row === undefined ? undefined : Object.values(row)[0];
+}
+/** A migrations directory that is misnumbered. Nothing has been applied when this is raised. */
+export class MigrationListError extends Error {
+    /**
+     * Creates a migration list error.
+     * @param message - What is wrong with the directory.
+     */
+    constructor(message) {
+        super(message);
+        this.name = "MigrationListError";
+    }
+}
+/** Pattern every migration file name must match: a zero-padded number, underscore, name, `.sql`. */
+export const MIGRATION_FILE = /^(\d{3})_[a-z0-9_]+\.sql$/;
+/**
+ * Reads and orders the migration files in a directory.
+ * @param dir - Directory containing `NNN_name.sql` files.
+ * @returns Migrations in version order, starting at 1 with no gaps.
+ * @throws {MigrationListError} If a `.sql` file is misnamed, a version repeats, or versions have gaps.
+ */
+export function listMigrations(dir) {
+    const migrations = [];
+    for (const name of readdirSync(dir).sort()) {
+        if (!name.endsWith(".sql")) {
+            continue;
+        }
+        const match = MIGRATION_FILE.exec(name);
+        // A misnamed file would silently never run, which is worse than refusing to start.
+        if (match === null) {
+            throw new MigrationListError(`migration file ${name} doesn't match NNN_name.sql`);
+        }
+        migrations.push({
+            version: Number(match[1]),
+            name,
+            sql: readFileSync(join(dir, name), "utf8"),
+        });
+    }
+    migrations.forEach((migration, index) => {
+        // Versions must be exactly 1..N: a duplicate or a gap means a lost or conflicting file.
+        if (migration.version !== index + 1) {
+            throw new MigrationListError(`migration versions must run 1..N without gaps or repeats; found ${migration.name} at position ${index + 1}`);
+        }
+    });
+    return migrations;
+}
+/** SQL creating the table that records applied migrations. */
+const MIGRATIONS_TABLE = `
+  CREATE TABLE IF NOT EXISTS schema_migrations (
+    version    INTEGER PRIMARY KEY,
+    name       TEXT NOT NULL,
+    applied_at TEXT NOT NULL
+  )`;
+/**
+ * Returns the highest applied migration version.
+ * @param db - An open database.
+ * @returns 0 for a new database, otherwise the last applied version.
+ */
+export function schemaVersion(db) {
+    db.exec(MIGRATIONS_TABLE);
+    const row = db
+        .prepare("SELECT COALESCE(MAX(version), 0) AS version FROM schema_migrations")
+        .get();
+    return row.version;
+}
+/**
+ * Applies every migration newer than the database's current version, each in its own transaction.
+ * @param db - An open database.
+ * @param migrations - Migrations from {@link listMigrations}.
+ * @param now - Clock for `applied_at`, injected for tests.
+ * @returns The versions applied by this call, in order; empty when already up to date.
+ * @throws {Error} Whatever SQLite raises for a failing migration. That migration is rolled back
+ *   completely, and earlier migrations from the same call stay applied.
+ */
+export function applyMigrations(db, migrations, now = () => new Date()) {
+    const current = schemaVersion(db);
+    const record = db.prepare("INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)");
+    const applied = [];
+    for (const migration of migrations.filter((m) => m.version > current)) {
+        // A transaction per migration: a failure leaves the schema at the last complete version.
+        transaction(db, () => {
+            db.exec(migration.sql);
+            record.run(migration.version, migration.name, now().toISOString());
+        });
+        applied.push(migration.version);
+    }
+    return applied;
+}
+/**
+ * Opens (creating if needed) the database and brings its schema up to date.
+ * @param path - Database file path, or `:memory:` for tests.
+ * @param migrationsDir - Directory of migration files.
+ * @returns An open database with WAL journaling, a busy timeout, and foreign keys enforced.
+ * @throws {MigrationListError} If the migrations directory is misnumbered.
+ * @throws {Error} Whatever SQLite raises while opening or migrating.
+ */
+export function openDatabase(path, migrationsDir) {
+    const migrations = listMigrations(migrationsDir);
+    if (path !== ":memory:") {
+        ensurePrivateDir(dirname(path));
+        // SQLite creates the -wal and -shm files with the database file's mode, so creating the file
+        // owner-only first keeps all three private (D-043). An existing file is tightened, never loosened.
+        ensurePrivateFile(path);
+        for (const file of [path, `${path}-wal`, `${path}-shm`]) {
+            tightenMode(file);
+        }
+    }
+    // The timeout is SQLite's busy timeout: it waits out short write locks instead of failing at once.
+    const db = new DatabaseSync(path, { timeout: 5000 });
+    // WAL lets the viewer read while an ingest writes.
+    db.exec("PRAGMA journal_mode = WAL");
+    // SQLite leaves foreign keys off unless asked, per connection. node:sqlite turns them on by default;
+    // saying so here keeps the requirement visible rather than inherited.
+    db.exec("PRAGMA foreign_keys = ON");
+    applyMigrations(db, migrations);
+    return db;
+}
+//# sourceMappingURL=database.js.map

package/dist/core/db/fingerprint.js

@@ -0,0 +1,71 @@
+/**
+ * @file Content fingerprints of database tables (docs/development.md P3.3, audit check A8).
+ *
+ * Proves re-runnability (D-002, `verify-against-ccusage` skill, check 4): fingerprint the derived
+ * tables, ingest again, and the fingerprints must be byte-identical. A fingerprint depends only on
+ * table contents, never on physical row order, so two databases holding the same rows agree even if
+ * the rows were inserted in a different order.
+ */
+import { createHash } from "node:crypto";
+import { blobBytes } from "./database.js";
+/** A table name that doesn't exist. Nothing is hashed when this is raised. */
+export class UnknownTableError extends Error {
+    /**
+     * Creates an unknown-table error.
+     * @param table - The name that wasn't found.
+     */
+    constructor(table) {
+        super(`no such table: ${table}`);
+        this.name = "UnknownTableError";
+    }
+}
+/**
+ * Serializes one SQLite value unambiguously.
+ * @param value - A value as `node:sqlite` returns it (a blob is a `Uint8Array`; a `Buffer` is one too).
+ * @returns A JSON-safe value. Blobs become `{"blob": "<hex>"}` so they can never collide with text,
+ *   and bigints become `{"int": "<decimal>"}` so large integers keep full precision.
+ */
+export function canonicalValue(value) {
+    if (value instanceof Uint8Array) {
+        return { blob: blobBytes(value).toString("hex") };
+    }
+    if (typeof value === "bigint") {
+        return { int: value.toString() };
+    }
+    return value;
+}
+/**
+ * Computes a SHA-256 fingerprint for each named table.
+ * @param db - An open database.
+ * @param tableNames - Tables to fingerprint.
+ * @returns Lowercase hex digests keyed by table name.
+ * @throws {UnknownTableError} If any table doesn't exist.
+ */
+export function fingerprintTables(db, tableNames) {
+    const existing = new Set(db.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all().map((row) => row.name));
+    // Check every name before hashing any, so a typo can't produce a partial result.
+    for (const table of tableNames) {
+        if (!existing.has(table)) {
+            throw new UnknownTableError(table);
+        }
+    }
+    const result = {};
+    for (const table of tableNames) {
+        // The name was validated against sqlite_master above; quoting guards unusual characters.
+        const rows = db.prepare(`SELECT * FROM "${table.replaceAll('"', '""')}"`);
+        // Arrays rather than objects: column order is part of the content, and names are not.
+        rows.setReturnArrays(true);
+        const values = rows.all();
+        // Sort serialized rows so the digest ignores physical order but not content.
+        const serialized = values.map((row) => JSON.stringify(row.map(canonicalValue))).sort();
+        const hash = createHash("sha256");
+        for (const row of serialized) {
+            hash.update(row);
+            // A separator makes row boundaries part of the digest: ["ab"] and ["a","b"] differ.
+            hash.update("\n");
+        }
+        result[table] = hash.digest("hex");
+    }
+    return result;
+}
+//# sourceMappingURL=fingerprint.js.map

package/dist/core/ingest/attribution.js

@@ -0,0 +1,131 @@
+/**
+ * @file Attributing working directories to repositories through git (docs/development.md P4.6, D-010, D-028).
+ *
+ * A repository is identified by git's common directory, not by the working directory's name. A
+ * worktree is a different directory but the same repository, and a subdirectory is not a separate
+ * project. ccusage (encoded directory names) and phuryn (last two path segments) both split one
+ * repository into several.
+ *
+ * Resolution depends on the filesystem today, not only on the logs, so results are cached in the
+ * `repositories` table, outside the tables that must rebuild identically from raw lines. Because the
+ * filesystem changes (folders are renamed and deleted), every run re-checks whether each cached
+ * directory and its repository root still exist, and resolves again only when that changed (D-028).
+ */
+import { spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { basename, dirname } from "node:path";
+import { transaction } from "../db/database.js";
+/**
+ * Runs the real git binary.
+ * @param args - Arguments after `git`.
+ * @returns Exit code (127 if git can't start) and trimmed stdout.
+ */
+export function runGit(args) {
+    const result = spawnSync("git", args, { encoding: "utf8" });
+    return { exitCode: result.status ?? 127, stdout: (result.stdout ?? "").trim() };
+}
+/**
+ * Asks git for the repository containing an existing directory.
+ * @param dir - An existing directory.
+ * @param git - Git runner.
+ * @returns The repository root, or null when git finds no repository.
+ */
+export function gitRoot(dir, git) {
+    // --path-format=absolute (git 2.31+) avoids resolving a relative ".git" against the wrong base.
+    const result = git(["-C", dir, "rev-parse", "--path-format=absolute", "--git-common-dir"]);
+    if (result.exitCode !== 0 || result.stdout === "") {
+        return null;
+    }
+    // A normal repository's common dir is <root>/.git, shared by every worktree. A bare repository's
+    // common dir is the repository itself.
+    const common = result.stdout;
+    return basename(common) === ".git" ? dirname(common) : common;
+}
+/**
+ * Resolves a working directory to its repository.
+ * @param cwd - A working directory recorded in a log line.
+ * @param lastKnownRoot - The repository root git gave while the directory still existed, if any.
+ * @param git - Git runner.
+ * @param exists - Existence check, injectable for tests.
+ * @returns For an existing directory, git's repository root or `not_git`. For a missing one
+ *   (D-028): the last-known root if given; otherwise the repository containing its nearest
+ *   existing parent folder; otherwise no root.
+ */
+export function resolveRepo(cwd, lastKnownRoot = null, git = runGit, exists = existsSync) {
+    if (exists(cwd)) {
+        const root = gitRoot(cwd, git);
+        return root === null ? { kind: "not_git" } : { kind: "repo", repoRoot: root };
+    }
+    // Git verified this root while the directory existed; that outranks any lookup from a parent.
+    if (lastKnownRoot !== null) {
+        return { kind: "missing", repoRoot: lastKnownRoot, via: "last_known" };
+    }
+    let parent = dirname(cwd);
+    // dirname of the filesystem root is the root itself, so the walk always ends.
+    while (!exists(parent) && dirname(parent) !== parent) {
+        parent = dirname(parent);
+    }
+    const root = exists(parent) ? gitRoot(parent, git) : null;
+    return root === null
+        ? { kind: "missing", repoRoot: null, via: "none" }
+        : { kind: "missing", repoRoot: root, via: "parent" };
+}
+/**
+ * Reports whether a cached row no longer matches the filesystem (D-028).
+ * @param row - The cached row.
+ * @param exists - Existence check.
+ * @returns True when the directory appeared or disappeared, or its root's existence changed.
+ */
+export function isStale(row, exists) {
+    if (exists(row.cwd) !== (row.kind !== "missing")) {
+        return true;
+    }
+    return row.repo_root !== null && exists(row.repo_root) !== (row.root_exists === 1);
+}
+/**
+ * Resolves working directories seen in log lines that have no cached resolution, and resolves
+ * again any cached one the filesystem no longer matches.
+ * @param db - Open, migrated database with derived tables up to date.
+ * @param now - Clock for `resolved_at`.
+ * @param resolve - Resolution function, given the last-known root; defaults to {@link resolveRepo}.
+ * @param exists - Existence check for re-validation and `root_exists`, injectable for tests.
+ * @returns Number of working directories resolved by this call (new plus re-resolved).
+ */
+export function resolveRepositories(db, now, resolve = (cwd, lastKnownRoot) => resolveRepo(cwd, lastKnownRoot), exists = existsSync) {
+    const pending = db
+        .prepare(`SELECT DISTINCT p.cwd FROM parsed_lines p
+       LEFT JOIN repositories r ON r.cwd = p.cwd
+       WHERE p.cwd IS NOT NULL AND r.cwd IS NULL
+       ORDER BY p.cwd`)
+        .all();
+    const stale = db
+        .prepare("SELECT cwd, kind, repo_root, root_exists, resolved_via FROM repositories ORDER BY cwd")
+        .all().filter((row) => isStale(row, exists));
+    const upsert = db.prepare(`INSERT INTO repositories (cwd, kind, repo_root, root_exists, resolved_via, resolved_at)
+     VALUES (?, ?, ?, ?, ?, ?)
+     ON CONFLICT (cwd) DO UPDATE SET kind = excluded.kind, repo_root = excluded.repo_root,
+       root_exists = excluded.root_exists, resolved_via = excluded.resolved_via,
+       resolved_at = excluded.resolved_at`);
+    /**
+     * Resolves one directory and stores the result.
+     * @param cwd - The working directory.
+     * @param lastKnownRoot - Its last git-verified root, if any.
+     */
+    const store = (cwd, lastKnownRoot) => {
+        const resolution = resolve(cwd, lastKnownRoot);
+        const root = resolution.kind === "not_git" ? null : resolution.repoRoot;
+        upsert.run(cwd, resolution.kind, root, root === null ? null : exists(root) ? 1 : 0, resolution.kind === "missing" ? resolution.via : "git", now().toISOString());
+    };
+    transaction(db, () => {
+        for (const { cwd } of pending) {
+            store(cwd, null);
+        }
+        for (const row of stale) {
+            // A root counts as last known only if git gave it: directly, or carried from an earlier git answer.
+            const verified = row.resolved_via === "git" || row.resolved_via === "last_known";
+            store(row.cwd, verified ? row.repo_root : null);
+        }
+    });
+    return pending.length + stale.length;
+}
+//# sourceMappingURL=attribution.js.map