@rvanbaalen/ofxreader 1.2.0

package/src/vendors/match.ts

@@ -0,0 +1,88 @@
+import type { Transaction } from "../model.ts";
+import { normalizeDescriptor } from "./normalize.ts";
+
+export type Candidate = {
+  normalized: string;
+  examples: string[]; // up to 3 raw descriptors that normalized to this
+  count: number; // how many transactions share this normalized descriptor
+  similarity: number; // Dice similarity to the query (0..1, 2 decimals)
+};
+
+/** The raw descriptor we treat as the vendor identity: name + memo + payee. */
+export function descriptorOf(t: Transaction): string {
+  return [t.name, t.memo, t.payee].filter((x) => x != null && x !== "").join(" ");
+}
+
+/** Confirmed match: a stored signature is a substring of the normalized descriptor. */
+export function confirmedMatch(t: Transaction, signatures: string[]): boolean {
+  if (signatures.length === 0) return false;
+  const norm = normalizeDescriptor(descriptorOf(t));
+  return signatures.some((sig) => sig !== "" && norm.includes(sig));
+}
+
+/** Sørensen–Dice similarity over character bigrams (0..1). No dependency. */
+export function diceSimilarity(a: string, b: string): number {
+  const A = bigrams(a);
+  const B = bigrams(b);
+  if (A.size === 0 || B.size === 0) return a === b ? 1 : 0;
+  let intersection = 0;
+  for (const [gram, countA] of A) {
+    const countB = B.get(gram);
+    if (countB != null) intersection += Math.min(countA, countB);
+  }
+  return (2 * intersection) / (sum(A) + sum(B));
+}
+
+/**
+ * Rank distinct normalized descriptors among `transactions` by their best Dice
+ * similarity to any of `queryTerms`, keeping those at or above `threshold`.
+ */
+export function rankCandidates(
+  transactions: Transaction[],
+  queryTerms: string[],
+  threshold: number,
+  limit = 10,
+): Candidate[] {
+  const groups = new Map<string, { examples: Set<string>; count: number }>();
+  for (const t of transactions) {
+    const raw = descriptorOf(t);
+    const norm = normalizeDescriptor(raw);
+    if (norm === "") continue;
+    const g = groups.get(norm) ?? { examples: new Set<string>(), count: 0 };
+    if (raw !== "" && g.examples.size < 3) g.examples.add(raw);
+    g.count += 1;
+    groups.set(norm, g);
+  }
+
+  const terms = queryTerms.filter((x) => x !== "");
+  const out: Candidate[] = [];
+  for (const [norm, g] of groups) {
+    let sim = 0;
+    for (const term of terms) sim = Math.max(sim, diceSimilarity(norm, term));
+    if (sim >= threshold) {
+      out.push({ normalized: norm, examples: [...g.examples], count: g.count, similarity: round2(sim) });
+    }
+  }
+  out.sort((a, b) => b.similarity - a.similarity || b.count - a.count);
+  return out.slice(0, limit);
+}
+
+function bigrams(s: string): Map<string, number> {
+  const t = s.replace(/\s+/g, " ").trim();
+  const m = new Map<string, number>();
+  for (let i = 0; i < t.length - 1; i++) {
+    const gram = t.slice(i, i + 2);
+    m.set(gram, (m.get(gram) ?? 0) + 1);
+  }
+  return m;
+}
+
+function sum(m: Map<string, number>): number {
+  let n = 0;
+  for (const v of m.values()) n += v;
+  return n;
+}
+
+function round2(n: number): number {
+  return Math.round(n * 100) / 100;
+}

package/src/vendors/normalize.ts

@@ -0,0 +1,35 @@
+/**
+ * Normalize a raw OFX transaction descriptor into a stable, comparable core.
+ *
+ *   "SQ *JASONS CARO 0123"  -> "JASONS CARO"
+ *   "TST* JASONSCAROUSEL"   -> "JASONSCAROUSEL"
+ *   "POS PURCHASE WHOLEFDS" -> "WHOLEFDS"
+ *
+ * Heuristic by design: confirmed signatures are exact substrings derived from real
+ * descriptors, so perfect normalization is not required — it mainly removes payment-
+ * processor noise so fuzzy candidate matching and substring matching behave sensibly.
+ */
+
+// Leading payment-processor "XX *" token, e.g. "SQ *", "TST*", "PP*", "IZ *".
+const STAR_PREFIX = /^[A-Z0-9]{2,5}\s*\*\s*/;
+
+// Conservative leading noise words (kept short to avoid clipping real names).
+const LEADING_NOISE = /^(POS|PURCHASE|DEBIT|CREDIT|CHECKCARD|RECURRING|ACH|WWW)\b[\s.]*/;
+
+export function normalizeDescriptor(input: string | null | undefined): string {
+  let s = (input ?? "").toUpperCase();
+
+  s = s.replace(STAR_PREFIX, " ").trimStart();
+
+  let prev = "";
+  while (s !== prev) {
+    prev = s;
+    s = s.replace(LEADING_NOISE, " ").trimStart();
+  }
+
+  // Drop digits and punctuation (store numbers, "#123", locations like "CA"
+  // survive as letters but rarely hurt substring matching).
+  s = s.replace(/[^A-Z\s]/g, " ");
+
+  return s.replace(/\s+/g, " ").trim();
+}

package/src/vendors/resolve.ts

@@ -0,0 +1,41 @@
+import type { Statement } from "../model.ts";
+import { filterTransactions } from "../query.ts";
+import type { TransactionFilters, QueryResult } from "../query.ts";
+import { confirmedMatch, rankCandidates } from "./match.ts";
+import type { Candidate } from "./match.ts";
+import { normalizeDescriptor } from "./normalize.ts";
+import { findVendorKey } from "./store.ts";
+import type { VendorStore } from "./store.ts";
+
+const SUGGEST_THRESHOLD = 0.6;
+
+export type VendorQueryResult = QueryResult & {
+  vendor: string; // canonical name (resolved key, or the query verbatim)
+  resolved: boolean; // was the vendor found in the store?
+  vendorCandidates: Candidate[]; // fuzzy, unconfirmed descriptors to propose
+};
+
+/**
+ * Resolve a vendor query: confirmed matches become the result (after the usual
+ * date/amount filters); the remaining descriptors are ranked as fuzzy candidates.
+ */
+export function resolveVendorQuery(
+  store: VendorStore,
+  statements: Statement[],
+  vendor: string,
+  filters: TransactionFilters,
+): VendorQueryResult {
+  const all = statements.flatMap((s) => s.transactions);
+  const key = findVendorKey(store, vendor);
+  const signatures = key != null ? (store.vendors[key]?.signatures ?? []) : [];
+
+  const confirmed = all.filter((t) => confirmedMatch(t, signatures));
+  const confirmedSet = new Set(confirmed);
+  const result = filterTransactions(confirmed, filters);
+
+  const remaining = all.filter((t) => !confirmedSet.has(t));
+  const queryTerms = [normalizeDescriptor(vendor), ...signatures];
+  const vendorCandidates = rankCandidates(remaining, queryTerms, SUGGEST_THRESHOLD);
+
+  return { ...result, vendor: key ?? vendor, resolved: key != null, vendorCandidates };
+}

package/src/vendors/store.ts

@@ -0,0 +1,93 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import { OfxError } from "../parser.ts";
+import { normalizeDescriptor } from "./normalize.ts";
+
+export type Vendor = {
+  signatures: string[]; // normalized, confirmed cores (for deterministic matching)
+  raw: string[]; // original confirmed descriptors (provenance)
+  updatedAt?: string; // YYYY-MM-DD
+};
+
+export type VendorStore = {
+  version: number;
+  vendors: Record<string, Vendor>;
+};
+
+/** Resolve the vendor store path: $OFXREADER_VENDORS, else $XDG_CONFIG_HOME, else ~/.config. */
+export function storePath(): string {
+  const override = process.env.OFXREADER_VENDORS;
+  if (override != null && override !== "") return override;
+  const xdg = process.env.XDG_CONFIG_HOME;
+  const base = xdg != null && xdg.trim() !== "" ? xdg : join(homedir(), ".config");
+  return join(base, "ofxreader", "vendors.json");
+}
+
+/** Load the store. A missing file is an empty store; corrupt JSON is an error. */
+export function load(path = storePath()): VendorStore {
+  if (!existsSync(path)) return { version: 1, vendors: {} };
+  let text: string;
+  try {
+    text = readFileSync(path, "utf8");
+  } catch (err) {
+    throw new OfxError("VENDOR_STORE_ERROR", `Could not read vendor store ${path}: ${(err as Error).message}`);
+  }
+  try {
+    const data = JSON.parse(text) as Partial<VendorStore>;
+    if (data == null || typeof data !== "object" || typeof data.vendors !== "object") {
+      throw new Error("expected an object with a `vendors` map");
+    }
+    return { version: data.version ?? 1, vendors: data.vendors as Record<string, Vendor> };
+  } catch (err) {
+    throw new OfxError("VENDOR_STORE_ERROR", `Vendor store ${path} is not valid: ${(err as Error).message}`);
+  }
+}
+
+/** Persist the store, creating the parent directory if needed. */
+export function save(store: VendorStore, path = storePath()): void {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, JSON.stringify(store, null, 2) + "\n");
+}
+
+/** Case-insensitive lookup of a vendor's canonical key. */
+export function findVendorKey(store: VendorStore, name: string): string | null {
+  const lower = name.toLowerCase();
+  for (const key of Object.keys(store.vendors)) {
+    if (key.toLowerCase() === lower) return key;
+  }
+  return null;
+}
+
+/**
+ * Add confirmed raw descriptors to a vendor (creating it if new), deriving
+ * normalized signatures. Mutates and returns the updated vendor entry.
+ */
+export function learn(
+  store: VendorStore,
+  name: string,
+  rawDescriptors: string[],
+  today: string,
+): Vendor {
+  const key = findVendorKey(store, name) ?? name;
+  const existing = store.vendors[key] ?? { signatures: [], raw: [] };
+  const signatures = new Set(existing.signatures);
+  const raw = new Set(existing.raw);
+
+  for (const descriptor of rawDescriptors) {
+    const trimmed = (descriptor ?? "").trim();
+    if (trimmed === "") continue;
+    raw.add(trimmed);
+    const signature = normalizeDescriptor(trimmed);
+    if (signature !== "") signatures.add(signature);
+  }
+
+  const vendor: Vendor = { signatures: [...signatures], raw: [...raw], updatedAt: today };
+  store.vendors[key] = vendor;
+  return vendor;
+}
+
+/** Today's date as YYYY-MM-DD (for `updatedAt`). */
+export function today(): string {
+  return new Date().toISOString().slice(0, 10);
+}

package/src/version.ts

@@ -0,0 +1,9 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+/** Read the package version from package.json (single source of truth). */
+export function getVersion(): string {
+  const pkgPath = fileURLToPath(new URL("../package.json", import.meta.url));
+  const pkg = JSON.parse(readFileSync(pkgPath, "utf8")) as { version?: string };
+  return pkg.version ?? "0.0.0";
+}