@morroc/open-ledger 0.14.1

package/dist/cli/commands/datasets.js

@@ -0,0 +1,56 @@
+import { emitList, fail, runAction } from "../output.js";
+import { listDatasets, readDataset, listDatasetNames, datasetHasKinds, } from "../../datasets/index.js";
+const DATASET_COLUMNS = [
+    { header: "Name", value: (d) => d.name },
+    { header: "Countries", value: (d) => d.countries.join(", ") },
+    { header: "Rows", value: (d) => String(d.rows), align: "right" },
+];
+const INSTITUTION_COLUMNS = [
+    { header: "Code", value: (r) => String(r.code ?? "") },
+    { header: "Label", value: (r) => String(r.label ?? "") },
+    { header: "Kind", value: (r) => String(r.kind ?? "") },
+    { header: "Country", value: (r) => r.country },
+];
+const DEFAULTS_COLUMNS = [
+    { header: "Country", value: (r) => r.country },
+    { header: "Locale", value: (r) => String(r.locale ?? "") },
+    { header: "Currency", value: (r) => String(r.currency ?? "") },
+];
+// Falls back to the generic country column so an unmapped dataset still renders something.
+const COLUMNS_BY_DATASET = {
+    institutions: INSTITUTION_COLUMNS,
+    defaults: DEFAULTS_COLUMNS,
+};
+const GENERIC_COLUMNS = [{ header: "Country", value: (r) => r.country }];
+function datasets(name, opts) {
+    if (name === undefined) {
+        if (opts.country || opts.kind) {
+            fail("USAGE", "--country/--kind need a dataset name (e.g. `oled datasets institutions --country th`)");
+        }
+        emitList(listDatasets(), DATASET_COLUMNS);
+        return;
+    }
+    const names = listDatasetNames();
+    if (!names.includes(name)) {
+        fail("NOT_FOUND", `unknown dataset "${name}"`, { hint: `known datasets: ${names.join(", ")}` });
+    }
+    if (opts.kind && !datasetHasKinds(name)) {
+        fail("USAGE", `dataset "${name}" has no kinds; drop --kind`);
+    }
+    const rows = readDataset(name, { country: opts.country, kind: opts.kind });
+    emitList(rows, COLUMNS_BY_DATASET[name] ?? GENERIC_COLUMNS);
+}
+export function registerDatasets(program) {
+    program
+        .command("datasets [name]")
+        .description("Reference datasets")
+        .option("--country <code>", "filter rows by country (e.g. th)")
+        .option("--kind <kind>", "filter institutions by kind")
+        .addHelpText("after", [
+        "",
+        "Behavior: read-only reference data, institution codes and per-country defaults behind account leaves.",
+        "Typical flow: bare datasets lists them; datasets institutions --country th filters a country's institutions.",
+        "Example: oled datasets institutions --country th --kind bank --json",
+    ].join("\n"))
+        .action(runAction(datasets));
+}

package/dist/cli/commands/doctor.js

@@ -0,0 +1,153 @@
+import chalk from "chalk";
+import { randomUUID } from "crypto";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs";
+import { join, resolve } from "path";
+import { getConfigPath, getDataDir } from "../../config.js";
+import { listMissingTables } from "../../db/schema.js";
+import { openDb } from "../db.js";
+import { getVersion } from "../../setup/install.js";
+import { SKILL_HOSTS } from "../../setup/hosts.js";
+import { EXIT, currentMode, emit, emitList, runAction } from "../output.js";
+import { errorMessage } from "../../lib/result.js";
+import { probeOcrEndpoint, resolveOcr } from "../../extract/ocr.js";
+const HARD_CHECKS = new Set(["db_open", "schema_tables_present"]);
+const REQUIRED_TABLES = ["accounts", "transactions", "questions"];
+function configCheck() {
+    return { name: "config_exists", ok: existsSync(getConfigPath()) };
+}
+async function dbOpenCheck() {
+    try {
+        const db = await openDb();
+        return { check: { name: "db_open", ok: true }, db };
+    }
+    catch (err) {
+        return { check: { name: "db_open", ok: false, detail: errorMessage(err) }, db: null };
+    }
+}
+function dataDirWritableCheck() {
+    try {
+        const dir = getDataDir();
+        mkdirSync(dir, { recursive: true });
+        const probe = join(dir, `.doctor-probe-${randomUUID()}`);
+        writeFileSync(probe, "ok");
+        rmSync(probe, { force: true });
+        return { name: "data_dir_writable", ok: true };
+    }
+    catch (err) {
+        return { name: "data_dir_writable", ok: false, detail: errorMessage(err) };
+    }
+}
+async function mupdfCheck() {
+    try {
+        await import("mupdf");
+        return { name: "mupdf_loads", ok: true };
+    }
+    catch (err) {
+        return { name: "mupdf_loads", ok: false, detail: errorMessage(err) };
+    }
+}
+function schemaTablesCheck(db) {
+    const name = "schema_tables_present";
+    if (!db)
+        return { name, ok: false, detail: "database not open" };
+    try {
+        const missing = listMissingTables(db, REQUIRED_TABLES);
+        return {
+            name,
+            ok: missing.length === 0,
+            detail: missing.length ? `missing: ${missing.join(", ")}` : undefined,
+        };
+    }
+    catch (err) {
+        return { name, ok: false, detail: errorMessage(err) };
+    }
+}
+export async function ocrEndpointCheck() {
+    const name = "ocr_endpoint";
+    const settings = resolveOcr();
+    if (!settings)
+        return { name, ok: true, detail: "not configured" };
+    const { baseUrl, model, preset } = settings;
+    const served = await probeOcrEndpoint(settings);
+    if (!served.ok)
+        return { name, ok: false, detail: `${baseUrl}: ${served.error}` };
+    if (!served.value.includes(model)) {
+        return {
+            name,
+            ok: false,
+            detail: `${baseUrl} does not serve ${model} (serving: ${served.value.join(", ") || "nothing"})`,
+        };
+    }
+    return { name, ok: true, detail: `${preset}/${model} at ${baseUrl}` };
+}
+async function runChecks() {
+    const checks = [];
+    checks.push(configCheck());
+    const { check: dbCheck, db } = await dbOpenCheck();
+    checks.push(dbCheck);
+    checks.push(dataDirWritableCheck());
+    checks.push(await mupdfCheck());
+    checks.push(schemaTablesCheck(db));
+    checks.push(skillPackCheck());
+    checks.push(await ocrEndpointCheck());
+    return checks;
+}
+/** Informational only; never a HARD_CHECK. */
+function skillPackCheck() {
+    const name = "skill_pack";
+    const candidates = [];
+    for (const host of SKILL_HOSTS) {
+        candidates.push({
+            host: host.id,
+            scope: "project",
+            path: join(resolve(process.cwd(), host.projectDir), "open-ledger", "VERSION"),
+        });
+    }
+    for (const host of SKILL_HOSTS) {
+        candidates.push({
+            host: host.id,
+            scope: "global",
+            path: join(host.globalDir(), "open-ledger", "VERSION"),
+        });
+    }
+    const found = candidates.find((c) => existsSync(c.path));
+    if (!found)
+        return { name, ok: true, detail: "not installed" };
+    const installed = readFileSync(found.path, "utf8").trim();
+    const where = `${found.host}, ${found.scope}`;
+    const cli = getVersion();
+    if (installed !== cli) {
+        return {
+            name,
+            ok: false,
+            detail: `installed ${installed} (${where}), cli ${cli} — refresh the skill (oled setup --force) or upgrade the CLI (npm install -g @morroc/open-ledger@latest)`,
+        };
+    }
+    return { name, ok: true, detail: `installed ${installed} (${where})` };
+}
+const CHECK_COLUMNS = [
+    { header: "Check", value: (r) => r.name },
+    { header: "OK", value: (r) => (r.ok ? "yes" : "no") },
+    { header: "Detail", value: (r) => r.detail ?? "" },
+];
+async function diagnoseEnvironment() {
+    const checks = await runChecks();
+    const ok = checks.filter((c) => HARD_CHECKS.has(c.name)).every((c) => c.ok);
+    const mode = currentMode();
+    if (mode.json) {
+        emit({ checks, ok });
+    }
+    else {
+        emitList(checks, CHECK_COLUMNS);
+        const line = `overall: ${ok ? "ready" : "not ready"}`;
+        process.stdout.write((mode.color ? (ok ? chalk.green(line) : chalk.red(line)) : line) + "\n");
+    }
+    if (!ok)
+        process.exitCode = EXIT.NOT_READY;
+}
+export function registerDoctor(program) {
+    program
+        .command("doctor")
+        .description("Diagnose the harness environment")
+        .action(runAction(diagnoseEnvironment));
+}

package/dist/cli/commands/files.js

@@ -0,0 +1,72 @@
+import { emitList, emitObject, fail, requireYes, runAction, } from "../output.js";
+import { openDb } from "../db.js";
+const FILE_COLUMNS = [
+    { header: "Status", value: (r) => r.status },
+    { header: "ID", value: (r) => r.id },
+    { header: "Source", value: (r) => r.source ?? "-" },
+    { header: "Ingested At", value: (r) => r.ingested_at ?? "-" },
+    { header: "Path", value: (r) => r.path },
+];
+/** files.status enum; `new` belongs to `ingest list`, not here — filtering by it here would silently match nothing. */
+const FILE_STATUSES = ["pending", "ingested", "failed"];
+async function listFiles(opts) {
+    // Checked up front — an unrecognized status would otherwise silently return zero rows.
+    const { status } = opts;
+    if (status !== undefined && !FILE_STATUSES.includes(status)) {
+        fail("USAGE", `--status must be one of ${FILE_STATUSES.join("|")}, got "${status}"`);
+    }
+    const db = await openDb();
+    const { listFiles: queryFiles } = await import("../../db/queries/files.js");
+    const rows = queryFiles(db);
+    emitList(status ? rows.filter((r) => r.status === status) : rows, FILE_COLUMNS);
+}
+async function showFile(id) {
+    const db = await openDb();
+    const { findFileById } = await import("../../db/queries/files.js");
+    const row = findFileById(db, id);
+    if (!row)
+        fail("NOT_FOUND", `no file: ${id}`);
+    const { countTransactionsBySourceFile } = await import("../../db/queries/transactions.js");
+    const { countQuestions } = await import("../../db/queries/questions.js");
+    emitObject({
+        type: "file_detail",
+        ...row,
+        transaction_count: countTransactionsBySourceFile(db, id),
+        open_question_count: countQuestions(db, { file_id: id }),
+    });
+}
+async function dropFile(id, opts) {
+    requireYes(opts, `dropping file ${id}`);
+    const db = await openDb();
+    const { deleteFile } = await import("../../db/queries/files.js");
+    const res = deleteFile(db, id);
+    if (!res.removed)
+        fail("NOT_FOUND", `no file: ${id}`);
+    // The extracted text describes a row that no longer exists, and nothing can
+    // reach it once the row is gone — same purge `ingest done`/`fail` do.
+    const { cleanCache } = await import("../../ingest/prepare.js");
+    const { removed } = cleanCache(id);
+    emitObject({
+        file_id: id,
+        removed_transactions: res.removedTransactions,
+        removed_questions: res.removedQuestions,
+        cache_removed: removed,
+    });
+}
+export function registerFiles(program) {
+    const files = program.command("files").description("Browse ingested files (list / show / drop)");
+    files
+        .command("list")
+        .description("List ingested files")
+        .option("--status <status>", `filter by status (${FILE_STATUSES.join("|")})`)
+        .action(runAction(listFiles));
+    files
+        .command("show <id>")
+        .description("Show a file with its transaction and open-question counts")
+        .action(runAction(showFile));
+    files
+        .command("drop <id>")
+        .description("Drop a file and cascade-remove its transactions/questions")
+        .option("--yes", "skip confirmation")
+        .action(runAction(dropFile));
+}

package/dist/cli/commands/ingest-commit.js

@@ -0,0 +1,279 @@
+import { EXIT, asRecord, currentMode, emit, emitObject, emitSummary, fail, readStdinBatch } from "../output.js";
+import { openDb } from "../db.js";
+import { newBatchId } from "../../lib/ids.js";
+import * as z from "zod";
+import { safeParse, str, num, json } from "../../lib/validate.js";
+// Delegates to the base hooks (so raise() still fires) while recording a typed event per side.
+function makeRecordingHooks(base, events) {
+    return {
+        onCommitted: (id) => base.onCommitted(id),
+        onDirtyInput: (input, reason) => {
+            base.onDirtyInput(input, reason);
+            events.push({ kind: "dirty", reason });
+        },
+        onUnknownMerchant: (input, id, attemptedId) => {
+            base.onUnknownMerchant(input, id, attemptedId);
+            events.push({ kind: "unknown_merchant", attemptedId });
+        },
+        onPlaceholderAccount: (side, accountId, id) => {
+            base.onPlaceholderAccount(side, accountId, id);
+            events.push({ kind: "placeholder", side, accountId });
+        },
+        onUncategorizedFallback: (side, accountId, id) => {
+            base.onUncategorizedFallback(side, accountId, id);
+            events.push({ kind: "uncategorized", side, accountId });
+        },
+        onSimilarAccount: (side, originalId, matchedId, id) => {
+            base.onSimilarAccount(side, originalId, matchedId, id);
+            events.push({ kind: "fuzzy", side, originalId, matchedId });
+        },
+        onCurrencyMismatch: (input, debit, credit) => {
+            base.onCurrencyMismatch(input, debit, credit);
+            events.push({ kind: "currency_mismatch" });
+        },
+    };
+}
+// Maps the resolver's own vocabulary (via the hooks) to the reported one.
+const SIDE_RESOLUTIONS = {
+    fuzzy: (event) => ({ resolved: event.matchedId, how: "fuzzy_matched" }),
+    placeholder: (event) => ({ resolved: event.accountId, how: "placeholder_created" }),
+    uncategorized: (event) => ({ resolved: event.accountId, how: "uncategorized_fallback" }),
+};
+// No event means the side resolved by exact match — every other route raises one.
+function classifySide(requested, side, events) {
+    const event = events.find((e) => "side" in e && e.side === side);
+    if (!event)
+        return { resolved: requested, how: "exact" };
+    return SIDE_RESOLUTIONS[event.kind](event);
+}
+// A duplicate re-commit fires no hooks and the stored row may have been recategorized
+// since, so report its stored accounts instead of re-deriving them.
+function reportSides(deps, outcome, raw, events) {
+    const stored = outcome.duplicate
+        ? deps.findTransactionById(deps.db, outcome.transactionId)
+        : null;
+    if (stored) {
+        return [
+            { side: "debit", requested: raw.debit_account_id, resolved: stored.debit_account_id, how: "as_committed" },
+            { side: "credit", requested: raw.credit_account_id, resolved: stored.credit_account_id, how: "as_committed" },
+        ];
+    }
+    return [
+        { side: "debit", requested: raw.debit_account_id, ...classifySide(raw.debit_account_id, "debit", events) },
+        { side: "credit", requested: raw.credit_account_id, ...classifySide(raw.credit_account_id, "credit", events) },
+    ];
+}
+function classifyMerchant(item, events, resolvedMerchantId) {
+    const hadMerchant = !!(item.merchant || item.merchant_id);
+    if (!hadMerchant)
+        return { how: "none" };
+    if (events.some((e) => e.kind === "unknown_merchant"))
+        return { how: "unknown" };
+    const mid = resolvedMerchantId();
+    return { how: "linked", merchant_id: mid ?? undefined };
+}
+// validateRawTransaction is the validity authority: missing fields default to "" and surface as dirty_input.
+// amount is excluded so its raw type reaches the validator's typeof check, unconverted by num().
+const LINKED_HEADER_SPEC = z.object({
+    date: str().default(""),
+    description: str().default(""),
+    raw_descriptor: str().nullable().default(null),
+    source_page: num().nullable().default(null),
+    merchant: json().nullable().default(null),
+    merchant_id: str().nullable().default(null),
+    group_id: str().nullable().default(null),
+    row_index: num().nullable().default(null),
+});
+const LINKED_LEG_SPEC = z.object({
+    debit_account_id: str().default(""),
+    credit_account_id: str().default(""),
+    currency: str().nullable().default(null),
+    description: str().optional(),
+    code: str().nullable().default(null),
+});
+const STANDALONE_SPEC = z.object({
+    id: str().optional(),
+    date: str().default(""),
+    description: str().default(""),
+    raw_descriptor: str().nullable().default(null),
+    source_page: num().nullable().default(null),
+    row_index: num().nullable().default(null),
+    merchant: json().nullable().default(null),
+    merchant_id: str().nullable().default(null),
+    debit_account_id: str().default(""),
+    credit_account_id: str().default(""),
+    currency: str().nullable().default(null),
+    code: str().nullable().default(null),
+});
+// debit/credit accept a snake_case synonym that isn't the camelCase auto-bridge.
+const LEG_ALIASES = {
+    debit_account_id: ["debit_account"],
+    credit_account_id: ["credit_account"],
+};
+// file_hash feeds deterministic transaction-id derivation elsewhere; memoized per file id.
+function makeFileHashCache(db, findFileById) {
+    const cache = new Map();
+    return (fileId) => {
+        if (!fileId)
+            return null;
+        if (!cache.has(fileId))
+            cache.set(fileId, findFileById(db, fileId)?.file_hash ?? null);
+        return cache.get(fileId) ?? null;
+    };
+}
+// A pre-pipeline reject (bad JSON shape) never throws — it reuses the per-row failure shape.
+function failRow(counters, index, message) {
+    return failOutcome(counters, index, { reason: "dirty_input", message, raisedQuestions: 0 });
+}
+function failOutcome(counters, index, outcome) {
+    counters.failed++;
+    return {
+        type: "result",
+        index,
+        ok: false,
+        reason: outcome.reason,
+        message: outcome.message,
+        raised_questions: outcome.raisedQuestions,
+    };
+}
+// Header + >=1 linked legs, committed atomically as one group.
+function commitCompoundRow(deps, row, linked) {
+    const { counters } = deps;
+    const parsedHeader = safeParse(LINKED_HEADER_SPEC, row.record);
+    if (!parsedHeader.ok)
+        return failRow(counters, row.index, parsedHeader.error);
+    const header = { ...parsedHeader.value, source_file_id: row.fileId };
+    const legs = [];
+    let legError;
+    for (const rawLeg of linked) {
+        const legRecord = asRecord(rawLeg);
+        if (!legRecord) {
+            legError = "each linked leg must be a JSON object.";
+            break;
+        }
+        const parsedLeg = safeParse(LINKED_LEG_SPEC, legRecord, { aliases: LEG_ALIASES });
+        if (!parsedLeg.ok) {
+            legError = parsedLeg.error;
+            break;
+        }
+        // Cast is a lie for malformed rows — validateRawTransaction rejects those.
+        legs.push({ ...parsedLeg.value, amount: legRecord.amount });
+    }
+    if (legError !== undefined)
+        return failRow(counters, row.index, legError);
+    const outcome = deps.commitLinkedTransactions(deps.db, row.ctx, header, legs, row.hooks);
+    counters.raisedTotal += outcome.raisedQuestions;
+    if (!outcome.ok)
+        return failOutcome(counters, row.index, outcome);
+    const allDuplicate = outcome.results.every((r) => r.duplicate);
+    if (allDuplicate)
+        counters.duplicates++;
+    else
+        counters.posted++;
+    return {
+        type: "result",
+        index: row.index,
+        ok: true,
+        group_id: outcome.group_id,
+        legs: outcome.results.map((r) => ({ transaction_id: r.id, duplicate: r.duplicate })),
+        duplicate: allDuplicate,
+        raised_questions: outcome.raisedQuestions,
+        merchant: classifyMerchant(parsedHeader.value, row.events, () => deps.findTransactionById(deps.db, outcome.results[0]?.id)?.merchant_id),
+    };
+}
+function commitStandaloneRow(deps, row) {
+    const { counters } = deps;
+    const parsed = safeParse(STANDALONE_SPEC, row.record, { aliases: LEG_ALIASES });
+    if (!parsed.ok)
+        return failRow(counters, row.index, parsed.error);
+    // Cast is a lie for malformed rows — validateRawTransaction rejects those.
+    const raw = {
+        ...parsed.value,
+        source_file_id: row.fileId,
+        amount: row.record.amount,
+    };
+    const outcome = deps.commitTransaction(deps.db, row.ctx, raw, row.hooks);
+    counters.raisedTotal += outcome.raisedQuestions;
+    if (!outcome.ok)
+        return failOutcome(counters, row.index, outcome);
+    if (outcome.duplicate)
+        counters.duplicates++;
+    else
+        counters.posted++;
+    return {
+        type: "result",
+        index: row.index,
+        ok: true,
+        transaction_id: outcome.transactionId,
+        duplicate: outcome.duplicate,
+        raised_questions: outcome.raisedQuestions,
+        merchant: classifyMerchant(parsed.value, row.events, () => deps.findTransactionById(deps.db, outcome.transactionId)?.merchant_id),
+        sides: reportSides(deps, outcome, raw, row.events),
+    };
+}
+export async function commitIngest(opts) {
+    const items = await readStdinBatch(opts.input);
+    if (items.length === 0)
+        fail("USAGE", "no transaction data provided");
+    const db = await openDb();
+    const { commitTransaction, commitLinkedTransactions, defaultTransactionCommitHooks } = await import("../../ingest/commit.js");
+    const { findTransactionById } = await import("../../db/queries/transactions.js");
+    const { findFileById } = await import("../../db/queries/files.js");
+    // An unknown --file id must fail before insert: a nulled file hash would break the
+    // deterministic transaction ids that dedup relies on.
+    if (opts.file && !findFileById(db, opts.file)) {
+        fail("NOT_FOUND", `no ingest entry: ${opts.file}`, {
+            hint: "run `oled ingest list --json` for the file ids, or drop --file to commit rows with no source file",
+        });
+    }
+    // Must be non-null: raise() no-ops when batchId is null, silently dropping every question.
+    const batchId = newBatchId();
+    const fileHashFor = makeFileHashCache(db, findFileById);
+    const counters = { posted: 0, duplicates: 0, failed: 0, raisedTotal: 0 };
+    const deps = {
+        db,
+        commitTransaction,
+        commitLinkedTransactions,
+        findTransactionById,
+        counters,
+    };
+    const results = [];
+    for (let index = 0; index < items.length; index++) {
+        const record = asRecord(items[index]);
+        if (!record) {
+            results.push(failRow(counters, index, "each transaction must be a JSON object."));
+            continue;
+        }
+        const fileId = (record.source_file_id ?? opts.file) ?? null;
+        const ctx = { batchId, fileId, fileHash: fileHashFor(fileId) };
+        const events = [];
+        const hooks = makeRecordingHooks(defaultTransactionCommitHooks(db, ctx), events);
+        const row = { record, index, fileId, ctx, events, hooks };
+        const linked = record.linked;
+        if (Array.isArray(linked) && linked.length > 0) {
+            results.push(commitCompoundRow(deps, row, linked));
+            continue;
+        }
+        results.push(commitStandaloneRow(deps, row));
+    }
+    const mode = currentMode();
+    if (mode.json) {
+        for (const r of results)
+            emit(r);
+        emitSummary({
+            batch_id: batchId,
+            posted: counters.posted,
+            duplicates: counters.duplicates,
+            failed: counters.failed,
+            raised_questions: counters.raisedTotal,
+        });
+    }
+    else {
+        for (const r of results)
+            emitObject(r);
+        process.stdout.write(`\nbatch ${batchId}: ${counters.posted} posted, ${counters.duplicates} duplicate(s), ${counters.failed} failed, ${counters.raisedTotal} question(s) raised\n`);
+    }
+    // Exit 7 only for genuine failures — duplicates are a successful no-op.
+    if (counters.failed > 0)
+        process.exitCode = EXIT.PARTIAL;
+}