@morroc/open-ledger 0.14.1

package/dist/db/queries/transactions.js

@@ -0,0 +1,319 @@
+import { accountExists } from "./accounts.js";
+import { upsertMerchant } from "./merchants.js";
+import { buildPatch } from "../../lib/patch.js";
+import { newGroupId, newTransactionId } from "../../lib/ids.js";
+import { ISO_DATE_RE } from "../../lib/date.js";
+/** Amount must already be an integer in minor units — this layer never sees decimals. */
+export function validateTransaction(input) {
+    if (!ISO_DATE_RE.test(input.date ?? "")) {
+        return { ok: false, reason: "Transaction date must be an ISO date (YYYY-MM-DD)." };
+    }
+    if (!input.description || !input.description.trim()) {
+        return { ok: false, reason: "Transaction description must not be empty." };
+    }
+    if (!Number.isInteger(input.amount) || input.amount <= 0) {
+        return { ok: false, reason: "Transaction amount must be a positive integer in minor units." };
+    }
+    if (!input.debit_account_id || !input.debit_account_id.trim()) {
+        return { ok: false, reason: "Transaction debit_account_id must not be empty." };
+    }
+    if (!input.credit_account_id || !input.credit_account_id.trim()) {
+        return { ok: false, reason: "Transaction credit_account_id must not be empty." };
+    }
+    if (input.debit_account_id === input.credit_account_id) {
+        return { ok: false, reason: "Transaction debit and credit accounts must differ." };
+    }
+    return { ok: true };
+}
+const INSERT_COLUMNS = "id, group_id, date, description, merchant_id, raw_descriptor, source_file_id, source_page, debit_account_id, credit_account_id, amount, currency, code, user_ref";
+function insertParams(id, merchantId, input) {
+    return [
+        id,
+        input.group_id ?? null,
+        input.date,
+        input.description,
+        merchantId,
+        input.raw_descriptor ?? null,
+        input.source_file_id ?? null,
+        input.source_page ?? null,
+        input.debit_account_id,
+        input.credit_account_id,
+        input.amount,
+        input.currency,
+        input.code ?? null,
+        input.user_ref ?? null,
+    ];
+}
+/**
+ * Validates, then `INSERT ... ON CONFLICT(id) DO NOTHING` — re-inserting the
+ * same derived id is a no-op. `duplicate` is true when the row already existed.
+ */
+export function insertTransaction(db, input) {
+    const check = validateTransaction(input);
+    if (!check.ok)
+        throw new Error(check.reason);
+    const id = input.id ?? newTransactionId();
+    let merchantId = input.merchant_id ?? null;
+    if (!merchantId && input.merchant) {
+        merchantId = upsertMerchant(db, input.merchant).id;
+    }
+    const result = db
+        .prepare(`INSERT INTO transactions (${INSERT_COLUMNS})
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(id) DO NOTHING`)
+        .run(...insertParams(id, merchantId, input));
+    return { id, duplicate: result.changes === 0 };
+}
+/**
+ * Insert several transactions sharing one group_id, atomically (any leg's
+ * failure rolls back all). group_id: `opts.group_id`, else the first input's, else a fresh `tg:`.
+ */
+export function insertLinkedTransactions(db, inputs, opts = {}) {
+    if (inputs.length === 0) {
+        throw new Error("insertLinkedTransactions requires at least one transaction.");
+    }
+    const groupId = opts.group_id ?? inputs.find((i) => i.group_id)?.group_id ?? newGroupId();
+    let results = [];
+    const tx = db.transaction(() => {
+        results = inputs.map((input) => insertTransaction(db, { ...input, group_id: groupId }));
+    });
+    tx();
+    return { results, group_id: groupId };
+}
+// Shared by listing and counts so a `--query` filter (reads joined names) matches in both.
+const LIST_FROM = `FROM transactions t
+   LEFT JOIN accounts da ON da.id = t.debit_account_id
+   LEFT JOIN accounts ca ON ca.id = t.credit_account_id
+   LEFT JOIN merchants m ON m.id = t.merchant_id`;
+const ROW_SELECT = `SELECT t.id, t.group_id, t.date, t.description, t.merchant_id,
+        t.raw_descriptor, t.source_file_id, t.source_page,
+        t.debit_account_id, t.credit_account_id, t.amount, t.currency,
+        t.code, t.user_ref, t.void_of, t.has_question, t.created_at,
+        da.name AS debit_account_name,
+        ca.name AS credit_account_name,
+        m.canonical_name AS merchant_name
+   ${LIST_FROM}`;
+/**
+ * Loads one transaction; when grouped, `group` carries every member (self
+ * included) ordered by id. Null if the id doesn't exist.
+ */
+export function findTransactionById(db, id) {
+    const row = db.prepare(`${ROW_SELECT} WHERE t.id = ?`).get(id);
+    if (!row)
+        return null;
+    if (!row.group_id)
+        return row;
+    const group = db
+        .prepare(`${ROW_SELECT} WHERE t.group_id = ? ORDER BY t.id`)
+        .all(row.group_id);
+    return { ...row, group };
+}
+// Shared by listTransactions/countTransactions so a filtered count matches the list.
+// `params` align positionally with the `?` placeholders in `whereSql`.
+function buildListWhere(opts) {
+    const conditions = [];
+    const params = [];
+    if (opts.account) {
+        conditions.push("(t.debit_account_id = ? OR t.credit_account_id = ?)");
+        params.push(opts.account, opts.account);
+    }
+    if (opts.from) {
+        conditions.push("t.date >= ?");
+        params.push(opts.from);
+    }
+    if (opts.to) {
+        conditions.push("t.date <= ?");
+        params.push(opts.to);
+    }
+    if (opts.query) {
+        conditions.push("(t.description LIKE ? OR t.raw_descriptor LIKE ? OR m.canonical_name LIKE ? OR da.name LIKE ? OR ca.name LIKE ?)");
+        const like = `%${opts.query}%`;
+        params.push(like, like, like, like, like);
+    }
+    if (opts.amount !== undefined) {
+        conditions.push("t.amount = ?");
+        params.push(opts.amount);
+    }
+    const whereSql = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
+    return { whereSql, params };
+}
+const DEFAULT_LIST_LIMIT = 50;
+const MAX_LIST_LIMIT = 500;
+/** Shared with the CLI summary so the reported cap matches the rows returned. */
+export function clampListLimit(limit) {
+    return Math.min(Math.max(limit ?? DEFAULT_LIST_LIMIT, 1), MAX_LIST_LIMIT);
+}
+export function listTransactions(db, opts = {}) {
+    const { whereSql, params } = buildListWhere(opts);
+    const limit = clampListLimit(opts.limit);
+    const rows = db
+        .prepare(`${ROW_SELECT} ${whereSql} ORDER BY t.date DESC, t.id DESC LIMIT ?`)
+        .all(...params, limit);
+    return opts.group ? clusterByGroup(rows) : rows;
+}
+/**
+ * Fold rows into per-group clusters, preserving the incoming (date DESC, id
+ * DESC) order for cluster first-appearance. Rows with a null group_id each
+ * become their own standalone cluster.
+ */
+function clusterByGroup(rows) {
+    const clusters = [];
+    const byGroup = new Map();
+    for (const row of rows) {
+        if (row.group_id == null) {
+            clusters.push({ group_id: null, transactions: [row] });
+            continue;
+        }
+        let cluster = byGroup.get(row.group_id);
+        if (!cluster) {
+            cluster = { group_id: row.group_id, transactions: [] };
+            byGroup.set(row.group_id, cluster);
+            clusters.push(cluster);
+        }
+        cluster.transactions.push(row);
+    }
+    return clusters;
+}
+export function deleteTransaction(db, id) {
+    return db.prepare(`DELETE FROM transactions WHERE id = ?`).run(id).changes > 0;
+}
+/**
+ * Re-points every matching transaction's `:from` side (debit OR credit) to
+ * `:to`. Rows whose other side already equals `:to` are skipped (would
+ * violate the debit<>credit CHECK) and counted in `skipped_self_transaction`.
+ */
+export function bulkRecategorize(db, filter, set) {
+    const from = filter.accountId;
+    const to = set.accountId;
+    if (!from)
+        throw new Error("bulkRecategorize: filter.accountId is required.");
+    if (!to)
+        throw new Error("bulkRecategorize: set.accountId is required.");
+    if (from === to) {
+        throw new Error("bulkRecategorize: set.accountId equals filter.accountId (no-op).");
+    }
+    if (!accountExists(db, to)) {
+        throw new Error(`bulkRecategorize: target account "${to}" does not exist.`);
+    }
+    const whereSql = "(t.debit_account_id = ? OR t.credit_account_id = ?)";
+    const params = [from, from];
+    let affected = 0;
+    let skipped = 0;
+    let sample = [];
+    const tx = db.transaction(() => {
+        const rows = db
+            .prepare(`SELECT t.id, t.debit_account_id, t.credit_account_id FROM transactions t WHERE ${whereSql}`)
+            .all(...params);
+        const toUpdate = [];
+        for (const r of rows) {
+            const other = r.debit_account_id === from ? r.credit_account_id : r.debit_account_id;
+            if (other === to) {
+                skipped++;
+                continue;
+            }
+            toUpdate.push(r.id);
+        }
+        if (toUpdate.length === 0)
+            return;
+        sample = toUpdate.slice(0, 10);
+        const placeholders = toUpdate.map(() => "?").join(",");
+        affected = db
+            .prepare(`UPDATE transactions
+           SET debit_account_id  = CASE WHEN debit_account_id  = ? THEN ? ELSE debit_account_id  END,
+               credit_account_id = CASE WHEN credit_account_id = ? THEN ? ELSE credit_account_id END
+         WHERE id IN (${placeholders})`)
+            .run(from, to, from, to, ...toUpdate).changes;
+    });
+    tx();
+    return { affected, skipped_self_transaction: skipped, sample_transaction_ids: sample };
+}
+/**
+ * The re-point step of `mergeAccounts` (src/accounts/accounts.ts). Rows that
+ * would become a degenerate self-transaction (one side `fromId`, the other
+ * already `toId`) are deleted FIRST — the debit<>credit CHECK forbids that state
+ * even transiently — then the remainder is re-pointed. Does not touch the
+ * accounts table.
+ */
+export function repointTransactions(db, fromId, toId) {
+    if (fromId === toId)
+        throw new Error("Cannot re-point transactions to the same account.");
+    let moved = 0;
+    let deletedSelfTransactions = 0;
+    const tx = db.transaction(() => {
+        deletedSelfTransactions = db
+            .prepare(`DELETE FROM transactions
+          WHERE (debit_account_id = ? AND credit_account_id = ?)
+             OR (credit_account_id = ? AND debit_account_id = ?)`)
+            .run(fromId, toId, fromId, toId).changes;
+        const d = db
+            .prepare(`UPDATE transactions SET debit_account_id = ? WHERE debit_account_id = ?`)
+            .run(toId, fromId).changes;
+        const c = db
+            .prepare(`UPDATE transactions SET credit_account_id = ? WHERE credit_account_id = ?`)
+            .run(toId, fromId).changes;
+        moved = d + c;
+    });
+    tx();
+    return { moved, deletedSelfTransactions };
+}
+/** Same filters as listTransactions, no limit; no opts counts every row (the case `status` uses). */
+export function countTransactions(db, opts = {}) {
+    const { whereSql, params } = buildListWhere(opts);
+    return db.prepare(`SELECT COUNT(*) AS n ${LIST_FROM} ${whereSql}`).get(...params).n;
+}
+export function countTransactionsBySourceFile(db, fileId) {
+    return db.prepare(`SELECT COUNT(*) AS n FROM transactions WHERE source_file_id = ?`).get(fileId).n;
+}
+/** The "account still in use" guard before deleting an account. */
+export function accountHasTransactions(db, accountId) {
+    return !!db
+        .prepare(`SELECT 1 FROM transactions WHERE debit_account_id = ? OR credit_account_id = ? LIMIT 1`)
+        .get(accountId, accountId);
+}
+const TRANSACTION_META_PATCH = {
+    date: {},
+    description: {},
+    merchant_id: {},
+    source_page: {},
+};
+/**
+ * Amount, currency, and the account columns are intentionally not accepted
+ * here: moving accounts is `bulkRecategorize`'s job, and amount/currency edits
+ * must go through delete + re-record to keep minor-unit invariants intact.
+ */
+export function updateTransactionMeta(db, id, fields) {
+    const { sets, params } = buildPatch(TRANSACTION_META_PATCH, {}, fields);
+    if (sets.length === 0)
+        return 0;
+    params.push(id);
+    return db.prepare(`UPDATE transactions SET ${sets.join(", ")} WHERE id = ?`).run(...params).changes;
+}
+/**
+ * Voids `fromId` into surviving `toId` (sets `void_of=toId`); never deletes,
+ * so re-ingesting the mirror's source file can't resurrect it. Requires
+ * matching amount/currency/both accounts but not date (statement vs. posting
+ * dates legitimately differ). Re-voiding an already-void row is a no-op.
+ */
+export function voidTransactionAsMirror(db, fromId, toId) {
+    if (fromId === toId)
+        throw new Error("Cannot merge a transaction into itself.");
+    const select = db.prepare(`SELECT amount, currency, debit_account_id, credit_account_id, void_of FROM transactions WHERE id = ?`);
+    const from = select.get(fromId);
+    if (!from)
+        throw new Error(`transaction "${fromId}" not found`);
+    const to = select.get(toId);
+    if (!to)
+        throw new Error(`transaction "${toId}" not found`);
+    if (from.void_of !== null)
+        return { alreadyVoid: true };
+    if (to.void_of !== null)
+        throw new Error(`cannot merge into voided transaction "${toId}"`);
+    if (from.amount !== to.amount ||
+        from.currency !== to.currency ||
+        from.debit_account_id !== to.debit_account_id ||
+        from.credit_account_id !== to.credit_account_id) {
+        throw new Error(`transactions "${fromId}" and "${toId}" are not mirrors (amount, currency, and both accounts must match)`);
+    }
+    db.prepare(`UPDATE transactions SET void_of = ? WHERE id = ?`).run(toId, fromId);
+    return { alreadyVoid: false };
+}

package/dist/db/queries/vault.js

@@ -0,0 +1,48 @@
+import { randomUUID } from "node:crypto";
+import { decryptSecret, encryptSecret } from "../encryption.js";
+// Most-used first: most likely to open the next statement too.
+const BY_USE = `ORDER BY use_count DESC, last_used_at DESC NULLS LAST, created_at ASC`;
+/** A vault browser, not a decrypt path — `listPasswordSecrets` is that. */
+export function listPasswords(db) {
+    return db
+        .prepare(`SELECT id, pattern, use_count, last_used_at FROM file_passwords ${BY_USE}`)
+        .all();
+}
+/** Every stored password, decrypted, most-used first. */
+export function listPasswordSecrets(db, dbKey) {
+    const rows = db
+        .prepare(`SELECT id, pattern, password_encrypted FROM file_passwords ${BY_USE}`)
+        .all();
+    return rows.map((row) => ({
+        id: row.id,
+        pattern: row.pattern,
+        password: decryptSecret(row.password_encrypted, dbKey),
+    }));
+}
+/** Replaces on conflict, so a bank's rotated password overwrites the stale one. */
+export function upsertPassword(db, pattern, password, dbKey) {
+    const encrypted = encryptSecret(password, dbKey);
+    const existing = db
+        .prepare(`SELECT id FROM file_passwords WHERE pattern = ?`)
+        .get(pattern);
+    if (existing) {
+        db.prepare(`UPDATE file_passwords
+       SET password_encrypted = ?, use_count = 0, last_used_at = NULL
+       WHERE id = ?`).run(encrypted, existing.id);
+        return existing.id;
+    }
+    const id = `fp:${randomUUID()}`;
+    db.prepare(`INSERT INTO file_passwords (id, pattern, password_encrypted) VALUES (?, ?, ?)`).run(id, pattern, encrypted);
+    return id;
+}
+export function recordPasswordUse(db, id) {
+    db.prepare(`UPDATE file_passwords
+     SET use_count = use_count + 1, last_used_at = datetime('now')
+     WHERE id = ?`).run(id);
+}
+export function deletePassword(db, patternOrId) {
+    const result = db
+        .prepare(`DELETE FROM file_passwords WHERE id = ? OR pattern = ?`)
+        .run(patternOrId, patternOrId);
+    return result.changes > 0;
+}

package/dist/db/schema.js

@@ -0,0 +1,105 @@
+import { copyFileSync, readdirSync, unlinkSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { MIGRATIONS } from "./migrations/index.js";
+/** Never drops tables or overwrites the file. */
+export function migrate(db, dbPath) {
+    applyMigrations(db, MIGRATIONS, dbPath);
+}
+/** Migrations are a parameter, not a direct import, so tests can drive a synthetic manifest. */
+export function applyMigrations(db, migrations, dbPath) {
+    const current = currentVersion(db);
+    if (current > migrations.length) {
+        throw new Error(`Database schema version ${current} is newer than this build supports ` +
+            `(${migrations.length}). Upgrade OpenLedger to open this database.`);
+    }
+    if (current === migrations.length)
+        return;
+    // Every migration is CREATE TABLE IF NOT EXISTS, so a foreign database would otherwise be half-adopted in silence.
+    if (current === 0 && hasUserTables(db)) {
+        const at = dbPath ? ` at ${dbPath}` : "";
+        throw new Error(`This database${at} is not an OpenLedger database. Your data has not been ` +
+            `touched. Back up the file, then remove it to start fresh.`);
+    }
+    // A version-0 database that reaches here is empty; only an upgrade has anything worth copying.
+    if (dbPath && current >= 1)
+        backupDatabase(db, dbPath);
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS schema_migrations (
+      version INTEGER PRIMARY KEY,
+      applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+  `);
+    for (let i = current; i < migrations.length; i++) {
+        const version = i + 1;
+        const migration = migrations[i];
+        const apply = db.transaction(() => {
+            migration.up(db);
+            db.prepare(`INSERT INTO schema_migrations (version) VALUES (?)`).run(version);
+        });
+        apply();
+    }
+}
+function currentVersion(db) {
+    const table = db
+        .prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'schema_migrations' LIMIT 1`)
+        .get();
+    if (!table)
+        return 0;
+    const row = db
+        .prepare(`SELECT COALESCE(MAX(version), 0) AS version FROM schema_migrations`)
+        .get();
+    return row.version;
+}
+/** Which of `tables` this database does not have — the schema half of `doctor`. */
+export function listMissingTables(db, tables) {
+    const placeholders = tables.map(() => "?").join(",");
+    const rows = db
+        .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (${placeholders})`)
+        .all(...tables);
+    const present = new Set(rows.map((r) => r.name));
+    return tables.filter((t) => !present.has(t));
+}
+/** True if the database holds any table other than the migration ledger itself. */
+function hasUserTables(db) {
+    const row = db
+        .prepare(`SELECT 1 FROM sqlite_master
+        WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name <> 'schema_migrations'
+        LIMIT 1`)
+        .get();
+    return !!row;
+}
+// Checkpoints the WAL first so the copy is complete; a non-WAL database
+// tolerates the checkpoint failing.
+function backupDatabase(db, dbPath) {
+    try {
+        db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+    }
+    catch {
+        // Rollback-journal databases have no WAL to checkpoint; the copy still holds.
+    }
+    copyFileSync(dbPath, `${dbPath}.${backupStamp()}.bak`);
+    pruneBackups(dbPath);
+}
+function backupStamp() {
+    const d = new Date();
+    const p = (n) => String(n).padStart(2, "0");
+    return (`${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}` +
+        `-${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}`);
+}
+/** Keeps the five newest `<dbPath>.<stamp>.bak` copies; older ones are pruned. */
+function pruneBackups(dbPath) {
+    const dir = dirname(dbPath);
+    const prefix = `${basename(dbPath)}.`;
+    // The fixed-width stamp makes lexicographic order chronological.
+    const backups = readdirSync(dir)
+        .filter((name) => name.startsWith(prefix) && name.endsWith(".bak"))
+        .sort();
+    for (const name of backups.slice(0, Math.max(0, backups.length - 5))) {
+        try {
+            unlinkSync(join(dir, name));
+        }
+        catch {
+            // A backup already gone is fine; pruning is best-effort.
+        }
+    }
+}

package/dist/extract/extract.js

@@ -0,0 +1,89 @@
+import { isServerFailure, ocrPages, } from "./ocr.js";
+import { probePdfPages, renderPdfPages, } from "./pdf.js";
+import { readerFor, verdictOf } from "./route.js";
+/** The neutral spec for the agent route, which hands rasters to an unknown vision model. */
+export const PAGE_RENDER = { dpi: 200, maxLongestDimPx: 1800 };
+const SERVER_FAILURE = {
+    unreachable: "ocr_unreachable",
+    rejected: "ocr_rejected",
+};
+// Images have no text layer; mupdf would resample a bare image as a fake 96-dpi PDF.
+async function probe(input) {
+    if (input.kind === "image")
+        return { ok: true, value: [] };
+    return probePdfPages(input.bytes);
+}
+async function pageImages(input, spec) {
+    if (input.kind === "image") {
+        return {
+            ok: true,
+            value: {
+                source: "original",
+                pages: [{ page: 1, mime: input.mime, bytes: input.bytes, path: input.path }],
+            },
+        };
+    }
+    const rendered = await renderPdfPages(input.bytes, spec);
+    if (!rendered.ok)
+        return rendered;
+    return { ok: true, value: { source: "raster", dpi: spec.dpi, pages: rendered.value } };
+}
+function placeholder(page) {
+    return `[page ${page}: OCR failed]`;
+}
+/** A failed page is a hole in the document, not a failed run — the caller reports `failedPages` and exits PARTIAL. */
+function ocrExtraction(outcomes, settings, textLayer) {
+    for (const outcome of outcomes) {
+        if (outcome.ok || !isServerFailure(outcome.reason))
+            continue;
+        return {
+            ok: false,
+            reason: SERVER_FAILURE[outcome.reason],
+            message: `page ${outcome.page}: ${outcome.message}`,
+        };
+    }
+    return {
+        ok: true,
+        value: {
+            kind: "text",
+            source: "ocr",
+            textLayer,
+            model: settings.model,
+            pages: outcomes.map((outcome) => ({
+                page: outcome.page,
+                text: outcome.ok ? outcome.text : placeholder(outcome.page),
+            })),
+            failedPages: outcomes.filter((outcome) => !outcome.ok).map((outcome) => outcome.page),
+        },
+    };
+}
+export async function extractFile(input, options) {
+    const probed = await probe(input);
+    if (!probed.ok)
+        return { ok: false, reason: "pdf_unreadable", message: probed.error };
+    // Each override cancels exactly one routing decision.
+    const overrides = options.overrides ?? {};
+    const ocr = overrides.noOcr ? null : options.ocr;
+    const textLayer = overrides.rescan ? "none" : verdictOf(probed.value);
+    const reader = readerFor(textLayer, ocr ? "ready" : "unset");
+    if (reader === "text-layer") {
+        return {
+            ok: true,
+            value: {
+                kind: "text",
+                source: "text-layer",
+                textLayer,
+                pages: probed.value.map(({ page, text }) => ({ page, text })),
+                failedPages: [],
+            },
+        };
+    }
+    // Reader chosen, so the resolution is too: the endpoint's preset when it reads, the neutral spec otherwise.
+    const images = await pageImages(input, ocr ? ocr.render : PAGE_RENDER);
+    if (!images.ok)
+        return { ok: false, reason: "pdf_unreadable", message: images.error };
+    // The READER table's "unset" column: no endpoint means reader "agent", images pass through as-is.
+    if (!ocr)
+        return { ok: true, value: { kind: "images", textLayer, ...images.value } };
+    return ocrExtraction(await ocrPages(images.value.pages, ocr), ocr, textLayer);
+}