@wirecat/cli-core 0.558.0 → 0.559.0

package/dist/store/sqlite/migrations.generated.js

@@ -932,5 +932,15 @@ export const GENERATED = [
             "CREATE TRIGGER facts_no_payment_bu BEFORE UPDATE OF type ON facts WHEN new.type = 'payment' BEGIN\n  SELECT RAISE(ABORT, 'facts: a payment is a payments row in store.finance, not a fact; upgrade the CLI');\nEND;",
             "INSERT OR REPLACE INTO store_settings (key, value, at) VALUES ('payment_facts_to_move', '1', unixepoch() * 1000);"
         ]
+    },
+    {
+        "name": "20261102192959_version-47-records",
+        "statements": [
+            "CREATE TABLE `record_relations` (\n\t`id` integer PRIMARY KEY,\n\t`record_id` integer NOT NULL,\n\t`role` text NOT NULL,\n\t`target_type` text NOT NULL,\n\t`target_external_id` text NOT NULL,\n\t`target_id` integer,\n\t`created_at` integer NOT NULL,\n\tCONSTRAINT `fk_record_relations_record_id_records_id_fk` FOREIGN KEY (`record_id`) REFERENCES `records`(`id`),\n\tCONSTRAINT `fk_record_relations_target_id_records_id_fk` FOREIGN KEY (`target_id`) REFERENCES `records`(`id`),\n\tCONSTRAINT `record_relations_record_id_role_target_type_target_external_id_unique` UNIQUE(`record_id`,`role`,`target_type`,`target_external_id`)\n);",
+            "CREATE TABLE `records` (\n\t`id` integer PRIMARY KEY,\n\t`account_id` integer NOT NULL,\n\t`type` text NOT NULL,\n\t`external_id` text NOT NULL,\n\t`fields` text NOT NULL,\n\t`revision` integer,\n\t`fingerprint` text NOT NULL,\n\t`entity_type` text,\n\t`entity_id` integer,\n\t`projected_fingerprint` text,\n\t`batch_id` text NOT NULL,\n\t`created_at` integer NOT NULL,\n\t`updated_at` integer NOT NULL,\n\t`deleted_at` integer,\n\tCONSTRAINT `fk_records_account_id_accounts_id_fk` FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`),\n\tCONSTRAINT `records_account_id_type_external_id_unique` UNIQUE(`account_id`,`type`,`external_id`)\n);",
+            "CREATE INDEX `record_relations_by_target_type_target_external_id` ON `record_relations` (`target_type`,`target_external_id`);",
+            "CREATE INDEX `record_relations_by_target_id` ON `record_relations` (`target_id`);",
+            "CREATE INDEX `records_by_entity_type_entity_id` ON `records` (`entity_type`,`entity_id`);"
+        ]
     }
 ];

package/dist/store/sqlite/records.d.ts

@@ -0,0 +1,81 @@
+import { type EnvelopeRecord, type EnvelopeSource, type IngestError } from "../../domain/records.js";
+import type { Author } from "./memories.js";
+import type { StoreContext } from "./open.js";
+export interface RecordKey {
+    provider: string;
+    account: string;
+    type: string;
+    externalId: string;
+}
+export interface SourceRecord extends RecordKey {
+    fields: Record<string, unknown>;
+    revision: number | null;
+    fingerprint: string;
+    relations: {
+        role: string;
+        type: string;
+        externalId: string;
+        resolved: boolean;
+    }[];
+    /** The entity made from it or mapped to it by the owner; `null` while it waits or for a type without a table. */
+    entity: string | null;
+    /** The source changed or deleted it after its entity was made: the owner should look at the entity. */
+    flagged: boolean;
+    batchId: string;
+    createdAt: string;
+    updatedAt: string;
+    deletedAt: string | null;
+}
+export interface RecordOutcome {
+    /** The record's place in the envelope's `records`. */
+    index: number;
+    key: string;
+    status: "created" | "updated" | "unchanged" | "deleted" | "refused";
+    error?: IngestError;
+}
+export interface RecordsWritten {
+    outcomes: RecordOutcome[];
+    /** `invoice:` and `payment:` refs this import proposed, also for records of earlier imports it completed. */
+    proposed: string[];
+    /** Kept, with no entity yet: the roles whose target is not imported or has no entity. */
+    waiting: {
+        index: number;
+        key: string;
+        missing: string[];
+    }[];
+    /** Changed or deleted at the source after its entity was made. */
+    flagged: {
+        index: number;
+        key: string;
+        entity: string;
+    }[];
+    /** A record of an earlier import this one completed, whose entity was still refused. */
+    laterErrors: IngestError[];
+}
+export interface RecordStore {
+    /** Writes validated records of one source, per chunk in one transaction; a dry run rolls everything back. */
+    write(source: EnvelopeSource, records: {
+        index: number;
+        record: EnvelopeRecord;
+    }[], options?: {
+        dryRun?: boolean;
+        chunk?: number;
+    }): Promise<RecordsWritten>;
+    get(key: RecordKey): Promise<SourceRecord | undefined>;
+    list(filter: {
+        provider: string;
+        account: string;
+        type?: string;
+        /** Only records whose entity waits for a reference. */
+        waiting?: boolean;
+        flagged?: boolean;
+        /** 1–500, 100 by default. */
+        limit?: number;
+    }): Promise<SourceRecord[]>;
+    /**
+     * The owner says which entity a record is: `person:4` for a payer a sheet names. Never done by name. Records that
+     * waited for it get their entity, as proposals.
+     */
+    map(key: RecordKey, entity: string, by: Author): Promise<SourceRecord>;
+}
+export declare const recordStoreOver: (context: StoreContext) => RecordStore;

package/dist/store/sqlite/records.js

@@ -0,0 +1,372 @@
+import { CliError } from "@wirecat/cli-base";
+import { formatDecimal, minorUnits, scaleOf } from "../../domain/money.js";
+import { pointerOf, recordFingerprint, } from "../../domain/records.js";
+import { accountPk, findAccountPk } from "./accounts.js";
+import { financeOperationsOver } from "./finance.js";
+import { requiredThing } from "./notes.js";
+import { referenceOfThing, storedThing } from "./things.js";
+import { read, write } from "./transaction.js";
+const PROJECTED = {
+    invoice: { roles: ["issuer", "counterparty", "project"], many: [] },
+    payment: { roles: ["payer", "payee", "project", "settles"], many: ["settles"] },
+};
+const isProjected = (type) => type === "invoice" || type === "payment";
+const IMPORTER = { bot: "import" };
+const REFERENCE_FIELDS = ["issuer", "counterparty", "project", "payer", "payee"];
+/** `at` names the record's member at fault; a refused entity is the fields' fault. */
+const refusal = (message, code, at) => {
+    throw new CliError("validation_error", message, { reason: code, ...(at === undefined ? {} : { at }) });
+};
+const codeOf = (error) => {
+    const reason = error.details?.reason;
+    if (typeof reason === "string")
+        return reason;
+    return error.code === "not_found" ? "record.reference-not-found" : "record.entity";
+};
+const iso = (value) => new Date(Number(value)).toISOString();
+const stringField = (fields, name) => {
+    const value = fields[name];
+    if (value === undefined || value === null)
+        return undefined;
+    if (typeof value !== "string")
+        return refusal(`fields.${name} is text`, "record.invalid");
+    return value;
+};
+const invoiceOf = (fields, related, source) => {
+    if (!Array.isArray(fields.lines) || fields.lines.some((line) => typeof line !== "object" || line === null))
+        return refusal("an invoice's fields.lines is a list of lines", "record.invalid");
+    const one = (role) => related.get(role)?.[0] ?? stringField(fields, role);
+    return {
+        ...written(source),
+        direction: fields.direction,
+        number: stringField(fields, "number"),
+        issuer: one("issuer"),
+        counterparty: one("counterparty"),
+        project: one("project"),
+        currency: fields.currency,
+        issueDate: stringField(fields, "issue_date"),
+        dueDate: stringField(fields, "due_date"),
+        total: fields.total,
+        lines: fields.lines.map((line) => ({
+            description: line.description,
+            quantity: line.quantity,
+            unitPrice: line.unit_price,
+            amount: line.amount,
+        })),
+    };
+};
+const paymentOf = (fields, related, source) => {
+    const one = (role) => related.get(role)?.[0] ?? stringField(fields, role);
+    return {
+        ...written(source),
+        direction: fields.direction,
+        payer: one("payer"),
+        payee: one("payee"),
+        project: one("project"),
+        amount: fields.amount,
+        currency: fields.currency,
+        date: fields.date,
+        reference: stringField(fields, "reference"),
+    };
+};
+const written = (source) => ({
+    by: IMPORTER,
+    source: `import:${source.provider}`,
+    reason: `imported from ${source.provider} account ${source.account}, batch ${source.batch_id}`,
+});
+export const recordStoreOver = (context) => {
+    const { database, now } = context;
+    const finance = financeOperationsOver(context);
+    const statements = new Map();
+    const sql = (text) => {
+        let statement = statements.get(text);
+        if (!statement) {
+            statement = database.prepare(text);
+            statements.set(text, statement);
+        }
+        return statement;
+    };
+    const entityRef = (type, id) => type == null ? null : (referenceOfThing(database, storedThing(type, id)) ?? `${String(type)}:${String(id)}`);
+    /** Roles a projection needs whose target has no entity yet, and the entities of the rest. */
+    const relatedOf = (recordId, type) => {
+        const { roles, many } = PROJECTED[type];
+        const rows = sql("SELECT rr.role, t.entity_type, t.entity_id FROM record_relations rr " +
+            "LEFT JOIN records t ON t.id = rr.target_id AND t.deleted_at IS NULL WHERE rr.record_id = ?").all(recordId);
+        const related = new Map();
+        const missing = new Set();
+        for (const row of rows) {
+            const role = String(row.role);
+            if (!roles.includes(role))
+                continue;
+            const ref = entityRef(row.entity_type, row.entity_id);
+            if (ref === null)
+                missing.add(role);
+            else
+                related.set(role, [...(related.get(role) ?? []), ref]);
+        }
+        for (const [role, refs] of related)
+            if (refs.length > 1 && !many.includes(role))
+                refusal(`a record has one ${role}`, "record.relation-ambiguous");
+        return { related, missing: [...missing] };
+    };
+    /** Makes the proposal a waiting invoice or payment record stands for, once every reference it names has an entity. */
+    const project = (row, source) => {
+        const type = row.type;
+        const { related, missing } = relatedOf(Number(row.id), type);
+        if (missing.length > 0)
+            return { missing };
+        const fields = JSON.parse(String(row.fields));
+        for (const name of REFERENCE_FIELDS)
+            stringField(fields, name);
+        const ref = type === "invoice" ? finance.addInvoice(invoiceOf(fields, related, source)).ref : settle(fields, related, source);
+        const entity = requiredThing(database, ref);
+        sql("UPDATE records SET entity_type = ?, entity_id = ?, projected_fingerprint = fingerprint WHERE id = ?").run(entity.type, entity.id, Number(row.id));
+        return { proposed: ref, missing: [] };
+    };
+    /** A payment settles each invoice it names as far as both allow, in the order named. */
+    const settle = (fields, related, source) => {
+        const made = finance.addPayment(paymentOf(fields, related, source));
+        const units = (amount) => minorUnits(amount, made.currency);
+        let left = units(made.unallocated);
+        for (const invoice of related.get("settles") ?? []) {
+            const share = Math.min(left, units(finance.invoice(invoice).open));
+            if (share <= 0)
+                continue;
+            const amount = formatDecimal(share, scaleOf(made.currency));
+            finance.allocate({ payment: made.ref, invoice, amount, by: IMPORTER });
+            left -= share;
+        }
+        return made.ref;
+    };
+    /** Records that waited on `target` and may now have their entity, and theirs in turn. */
+    const completeDependents = (target, source, proposed, laterErrors) => {
+        const queue = [target];
+        while (queue.length > 0) {
+            const rows = sql("SELECT DISTINCT r.* FROM record_relations rr JOIN records r ON r.id = rr.record_id WHERE rr.target_id = ? " +
+                "AND r.entity_id IS NULL AND r.deleted_at IS NULL AND r.type IN ('invoice', 'payment')").all(queue.shift());
+            for (const row of rows) {
+                try {
+                    const done = write(database, () => project(row, source));
+                    if (done.proposed) {
+                        proposed.push(done.proposed);
+                        queue.push(Number(row.id));
+                    }
+                }
+                catch (error) {
+                    if (!(error instanceof CliError))
+                        throw error;
+                    laterErrors.push({
+                        pointer: "",
+                        code: codeOf(error),
+                        message: `${String(row.type)} ${String(row.external_id)} waits: ${error.message}`,
+                    });
+                }
+            }
+        }
+    };
+    const relate = (recordId, accountId, relations, at) => {
+        sql("DELETE FROM record_relations WHERE record_id = ?").run(recordId);
+        for (const relation of relations)
+            sql("INSERT OR IGNORE INTO record_relations (record_id, role, target_type, target_external_id, target_id, created_at) " +
+                "VALUES (?, ?, ?, ?, (SELECT id FROM records WHERE account_id = ? AND type = ? AND external_id = ?), ?)").run(recordId, relation.role, relation.type, relation.external_id, accountId, relation.type, relation.external_id, at);
+    };
+    const writeOne = (accountId, source, { record }, proposed) => {
+        const stored = sql("SELECT * FROM records WHERE account_id = ? AND type = ? AND external_id = ?").get(accountId, record.type, record.external_id);
+        const relations = record.relations ?? [];
+        const fingerprint = record.operation === "upsert" ? recordFingerprint(record.fields ?? {}, relations) : null;
+        const stale = (why) => refusal(why, "record.stale-revision", "revision");
+        if (stored && stored.revision != null) {
+            const held = Number(stored.revision);
+            if (record.revision === undefined)
+                stale(`the store holds revision ${held}: send the record's revision`);
+            else if (record.revision < held)
+                stale(`revision ${record.revision} is older than the stored ${held}`);
+            else if (record.revision === held && fingerprint !== null && fingerprint !== stored.fingerprint)
+                stale(`revision ${held} is stored with other content: a change needs a higher revision`);
+        }
+        const time = now();
+        const revision = record.revision ?? (stored?.revision == null ? null : Number(stored.revision));
+        if (record.operation === "delete") {
+            if (!stored)
+                return refusal(`no ${record.type} ${record.external_id} to delete`, "record.not-found", "external_id");
+            if (stored.deleted_at != null)
+                return { status: "unchanged", id: Number(stored.id) };
+            sql("UPDATE records SET deleted_at = ?, updated_at = ?, revision = ?, batch_id = ? WHERE id = ?").run(time, time, revision, source.batch_id, Number(stored.id));
+            return { status: "deleted", id: Number(stored.id), flagged: stored.entity_id != null };
+        }
+        const fields = JSON.stringify(record.fields);
+        if (stored && stored.fingerprint === fingerprint && stored.deleted_at == null)
+            return { status: "unchanged", id: Number(stored.id) };
+        const id = stored
+            ? Number(stored.id)
+            : Number(sql("INSERT INTO records (account_id, type, external_id, fields, revision, fingerprint, batch_id, created_at, " +
+                "updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id").get(accountId, record.type, record.external_id, fields, revision, fingerprint, source.batch_id, time, time)
+                ?.id);
+        if (stored)
+            sql("UPDATE records SET fields = ?, revision = ?, fingerprint = ?, batch_id = ?, updated_at = ?, deleted_at = NULL " +
+                "WHERE id = ?").run(fields, revision, fingerprint, source.batch_id, time, id);
+        else
+            sql("UPDATE record_relations SET target_id = ? WHERE target_id IS NULL AND target_type = ? AND target_external_id = ? " +
+                "AND record_id IN (SELECT id FROM records WHERE account_id = ?)").run(id, record.type, record.external_id, accountId);
+        relate(id, accountId, relations, time);
+        let projected = false;
+        if (isProjected(record.type) && stored?.entity_id == null) {
+            const done = project({ id, type: record.type, fields, account_id: accountId }, source);
+            if (done.proposed)
+                proposed.push(done.proposed);
+            projected = done.proposed !== undefined;
+        }
+        return {
+            status: stored ? "updated" : "created",
+            id,
+            flagged: stored?.entity_id != null && stored.projected_fingerprint !== fingerprint,
+            // Records waiting on this one move only when it has an entity: made now, or back from a delete.
+            complete: projected || (stored?.deleted_at != null && stored.entity_id != null),
+        };
+    };
+    const missingOf = (id) => {
+        const row = sql("SELECT type, entity_id, deleted_at FROM records WHERE id = ?").get(id);
+        if (!row || row.entity_id != null || row.deleted_at != null || !isProjected(row.type))
+            return null;
+        const { missing } = relatedOf(id, row.type);
+        return missing.length > 0 ? missing : null;
+    };
+    const ingest = (source, records, chunk) => {
+        const outcomes = [];
+        const proposed = [];
+        const flagged = [];
+        const laterErrors = [];
+        const kept = [];
+        for (let start = 0; start < records.length; start += chunk) {
+            write(database, () => {
+                const accountId = accountPk(context, { provider: source.provider, account: source.account });
+                const complete = [];
+                for (const item of records.slice(start, start + chunk)) {
+                    try {
+                        const done = isProjected(item.record.type)
+                            ? write(database, () => writeOne(accountId, source, item, proposed))
+                            : writeOne(accountId, source, item, proposed);
+                        outcomes.push({ index: item.index, key: item.record.key, status: done.status });
+                        if (done.id === undefined)
+                            continue;
+                        kept.push({ index: item.index, key: item.record.key, id: done.id });
+                        if (done.flagged) {
+                            const row = sql("SELECT entity_type, entity_id FROM records WHERE id = ?").get(done.id);
+                            flagged.push({
+                                index: item.index,
+                                key: item.record.key,
+                                entity: entityRef(row?.entity_type, row?.entity_id),
+                            });
+                        }
+                        if (done.complete)
+                            complete.push(done.id);
+                    }
+                    catch (error) {
+                        if (!(error instanceof CliError))
+                            throw error;
+                        const at = error.details?.at ?? "fields";
+                        const pointer = pointerOf("records", item.index, at);
+                        outcomes.push({
+                            index: item.index,
+                            key: item.record.key,
+                            status: "refused",
+                            error: { pointer, code: codeOf(error), message: error.message, key: item.record.key },
+                        });
+                    }
+                }
+                for (const id of complete)
+                    completeDependents(id, source, proposed, laterErrors);
+            });
+        }
+        const waiting = read(database, () => kept.flatMap(({ index, key, id }) => {
+            const missing = missingOf(id);
+            return missing === null ? [] : [{ index, key, missing }];
+        }));
+        return { outcomes, proposed: [...new Set(proposed)], waiting, flagged, laterErrors };
+    };
+    const sourceRecordOf = (row) => ({
+        provider: String(row.provider),
+        account: String(row.account),
+        type: String(row.type),
+        externalId: String(row.external_id),
+        fields: JSON.parse(String(row.fields)),
+        revision: row.revision == null ? null : Number(row.revision),
+        fingerprint: String(row.fingerprint),
+        relations: sql("SELECT role, target_type, target_external_id, target_id FROM record_relations WHERE record_id = ? ORDER BY id")
+            .all(Number(row.id))
+            .map((relation) => ({
+            role: String(relation.role),
+            type: String(relation.target_type),
+            externalId: String(relation.target_external_id),
+            resolved: relation.target_id != null,
+        })),
+        entity: entityRef(row.entity_type, row.entity_id),
+        flagged: row.entity_id != null && (row.deleted_at != null || row.projected_fingerprint !== row.fingerprint),
+        batchId: String(row.batch_id),
+        createdAt: iso(row.created_at),
+        updatedAt: iso(row.updated_at),
+        deletedAt: row.deleted_at == null ? null : iso(row.deleted_at),
+    });
+    const SELECT = "SELECT r.*, a.provider, a.external_id AS account FROM records r JOIN accounts a ON a.id = r.account_id ";
+    const rowOf = (key) => sql(`${SELECT}WHERE a.provider = ? AND a.external_id = ? AND r.type = ? AND r.external_id = ?`).get(key.provider, key.account, key.type, key.externalId);
+    return {
+        write: async (source, records, { dryRun = false, chunk = 1000 } = {}) => {
+            if (!dryRun)
+                return ingest(source, records, chunk);
+            let result;
+            const rollback = new Error("dry run");
+            try {
+                write(database, () => {
+                    result = ingest(source, records, chunk);
+                    throw rollback;
+                });
+            }
+            catch (error) {
+                if (error !== rollback)
+                    throw error;
+            }
+            return result;
+        },
+        get: async (key) => read(database, () => {
+            const row = rowOf(key);
+            return row ? sourceRecordOf(row) : undefined;
+        }),
+        list: async ({ provider, account, type, waiting, flagged, limit = 100 }) => {
+            if (!Number.isSafeInteger(limit) || limit < 1 || limit > 500)
+                throw new CliError("validation_error", "a record list takes 1–500 items");
+            return read(database, () => {
+                if (findAccountPk(context, { provider, account }) === undefined)
+                    return [];
+                const rows = database
+                    .prepare(`${SELECT}WHERE a.provider = ? AND a.external_id = ? AND (? IS NULL OR r.type = ?) ` +
+                    "AND (? IS NULL OR (r.entity_id IS NULL AND r.deleted_at IS NULL AND r.type IN ('invoice', 'payment'))) " +
+                    "AND (? IS NULL OR (r.entity_id IS NOT NULL AND (r.deleted_at IS NOT NULL " +
+                    "OR r.projected_fingerprint IS NOT r.fingerprint))) ORDER BY r.updated_at DESC, r.id DESC LIMIT ?")
+                    .all(provider, account, type ?? null, type ?? null, waiting ? 1 : null, flagged ? 1 : null, limit);
+                return rows.map(sourceRecordOf);
+            });
+        },
+        map: async (key, entity, by) => {
+            if (by !== "owner")
+                throw new CliError("permission_error", "only the owner says which entity a record is", {
+                    reason: "record.map-owner-only",
+                });
+            const proposed = [];
+            const laterErrors = [];
+            write(database, () => {
+                const row = rowOf(key);
+                if (!row || row.deleted_at != null)
+                    throw new CliError("not_found", `no ${key.type} ${key.externalId} from ${key.provider} ${key.account}`);
+                const thing = requiredThing(database, entity, "the entity");
+                if (thing.type !== key.type)
+                    refusal(`a ${key.type} record maps to a ${key.type}:, not ${entity}`, "record.map-type");
+                if (row.entity_id != null && !(row.entity_type === thing.type && Number(row.entity_id) === thing.id))
+                    refusal(`${key.type} ${key.externalId} is already ${entityRef(row.entity_type, row.entity_id)}`, "record.mapped");
+                sql("UPDATE records SET entity_type = ?, entity_id = ?, projected_fingerprint = fingerprint WHERE id = ?").run(thing.type, thing.id, Number(row.id));
+                const source = { provider: key.provider, account: key.account, batch_id: String(row.batch_id) };
+                completeDependents(Number(row.id), source, proposed, laterErrors);
+            });
+            return read(database, () => sourceRecordOf(rowOf(key)));
+        },
+    };
+};