@crewhaus/tool-table 0.7.0

package/dist/lib/reconcile.js

@@ -0,0 +1,336 @@
+/**
+ * Reconciling two exports, and matching records that do not share a key.
+ *
+ * "What changed between yesterday's file and today's" and "is this the same
+ * customer as that one" are the two questions that make a person open two
+ * spreadsheets side by side. Both are mechanical, and both are done badly by
+ * eye at any real size.
+ */
+const keyOf = (row, columns) => JSON.stringify(columns.map((c) => row[c] ?? ""));
+/**
+ * Reconcile two tables by key.
+ *
+ * Duplicate keys are reported rather than silently resolved. With a
+ * duplicated key there is no fact of the matter about which row changed into
+ * which, and picking one produces a diff that looks authoritative and is
+ * arbitrary.
+ */
+export function diffTables(before, after, keyColumns, ignoreColumns = []) {
+    if (keyColumns.length === 0)
+        throw new Error("a diff needs at least one key column");
+    const beforeColumns = new Set(before.flatMap((r) => Object.keys(r)));
+    const afterColumns = new Set(after.flatMap((r) => Object.keys(r)));
+    for (const column of keyColumns) {
+        if (before.length > 0 && !beforeColumns.has(column)) {
+            throw new Error(`the key column "${column}" is not in the before table`);
+        }
+        if (after.length > 0 && !afterColumns.has(column)) {
+            throw new Error(`the key column "${column}" is not in the after table`);
+        }
+    }
+    const index = (rows, side) => {
+        const map = new Map();
+        const duplicates = [];
+        const counts = new Map();
+        for (const row of rows) {
+            const key = keyOf(row, keyColumns);
+            counts.set(key, (counts.get(key) ?? 0) + 1);
+            if (!map.has(key))
+                map.set(key, row);
+        }
+        for (const [key, count] of counts)
+            if (count > 1)
+                duplicates.push({ key, side, count });
+        return { map, duplicates };
+    };
+    const a = index(before, "before");
+    const b = index(after, "after");
+    const ignored = new Set([...ignoreColumns, ...keyColumns]);
+    const compared = [...new Set([...beforeColumns, ...afterColumns])]
+        .filter((c) => !ignored.has(c))
+        .sort();
+    const added = [];
+    const removed = [];
+    const changed = [];
+    let unchanged = 0;
+    for (const [key, row] of b.map) {
+        const prior = a.map.get(key);
+        if (prior === undefined) {
+            added.push(row);
+            continue;
+        }
+        const changes = compared
+            .filter((column) => (prior[column] ?? "") !== (row[column] ?? ""))
+            .map((column) => ({ column, from: prior[column] ?? "", to: row[column] ?? "" }));
+        if (changes.length === 0)
+            unchanged++;
+        else
+            changed.push({ key, changes });
+    }
+    for (const [key, row] of a.map)
+        if (!b.map.has(key))
+            removed.push(row);
+    return {
+        keyColumns,
+        added,
+        removed,
+        changed,
+        unchanged,
+        duplicateKeys: [...a.duplicates, ...b.duplicates],
+        columnsOnlyInBefore: [...beforeColumns].filter((c) => !afterColumns.has(c)).sort(),
+        columnsOnlyInAfter: [...afterColumns].filter((c) => !beforeColumns.has(c)).sort(),
+    };
+}
+const NAME_NOISE = new Set([
+    "mr",
+    "mrs",
+    "ms",
+    "miss",
+    "dr",
+    "prof",
+    "sir",
+    "jr",
+    "sr",
+    "ii",
+    "iii",
+    "iv",
+    "phd",
+    "md",
+]);
+const COMPANY_NOISE = new Set([
+    "inc",
+    "llc",
+    "ltd",
+    "limited",
+    "corp",
+    "corporation",
+    "co",
+    "company",
+    "gmbh",
+    "sa",
+    "sas",
+    "bv",
+    "nv",
+    "plc",
+    "ag",
+    "pty",
+    "srl",
+    "spa",
+    "oy",
+    "ab",
+    "as",
+]);
+/**
+ * Canonicalize a contact's identifying fields.
+ *
+ * Gmail's dots and `+tags` are folded ONLY for gmail-family domains, because
+ * they are a Gmail feature and not a rule of email: treating `a.b@other.com`
+ * as `ab@other.com` merges two different people. The original is never
+ * discarded — this returns a comparison key beside it, and any fold that
+ * happened is named in `notes`.
+ */
+export function normalizeContact(input) {
+    const notes = [];
+    let email = (input.email ?? "").trim().toLowerCase();
+    let emailDomain = "";
+    if (email.includes("@")) {
+        const at = email.lastIndexOf("@");
+        let local = email.slice(0, at);
+        emailDomain = email.slice(at + 1);
+        const plus = local.indexOf("+");
+        if (plus > 0) {
+            local = local.slice(0, plus);
+            notes.push("dropped an email +tag");
+        }
+        if (["gmail.com", "googlemail.com"].includes(emailDomain)) {
+            const withoutDots = local.split(".").join("");
+            if (withoutDots !== local)
+                notes.push("folded gmail dots");
+            local = withoutDots;
+            if (emailDomain === "googlemail.com")
+                emailDomain = "gmail.com";
+        }
+        email = `${local}@${emailDomain}`;
+    }
+    else if (email !== "") {
+        notes.push("the email has no @, so it was left as written");
+    }
+    // Phones keep only digits, and a leading + is preserved as the sign of an
+    // international number. A national number is prefixed only when the caller
+    // said which country to assume.
+    const rawPhone = (input.phone ?? "").trim();
+    let phone = "";
+    if (rawPhone !== "") {
+        const international = rawPhone.trimStart().startsWith("+");
+        const digits = rawPhone.replace(/\D/g, "");
+        if (international)
+            phone = `+${digits}`;
+        else if (digits.startsWith("00"))
+            phone = `+${digits.slice(2)}`;
+        else if (input.defaultCountryCode !== undefined && input.defaultCountryCode !== "") {
+            const trimmed = digits.replace(/^0+/, "");
+            phone = `+${input.defaultCountryCode.replace(/\D/g, "")}${trimmed}`;
+            notes.push(`assumed country code ${input.defaultCountryCode}`);
+        }
+        else {
+            phone = digits;
+            notes.push("no country code, so the number is national and may not compare across regions");
+        }
+    }
+    const name = (input.name ?? "").trim().replace(/\s+/g, " ");
+    const nameKey = name
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s]/gu, " ")
+        .split(/\s+/)
+        .filter((part) => part !== "" && !NAME_NOISE.has(part))
+        .sort()
+        .join(" ");
+    const company = (input.company ?? "").trim().replace(/\s+/g, " ");
+    const companyKey = company
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s]/gu, " ")
+        .split(/\s+/)
+        .filter((part) => part !== "" && !COMPANY_NOISE.has(part))
+        .join(" ");
+    return { email, emailDomain, phone, name, nameKey, company: companyKey, notes };
+}
+// ---------------------------------------------------------------------------
+/** Levenshtein distance, bounded so a long pair cannot become quadratic cost. */
+export function editDistance(a, b, max = 64) {
+    if (a === b)
+        return 0;
+    if (Math.abs(a.length - b.length) > max)
+        return max + 1;
+    const s = a.length > b.length ? b : a;
+    const t = a.length > b.length ? a : b;
+    let previous = Array.from({ length: s.length + 1 }, (_, i) => i);
+    for (let i = 1; i <= t.length; i++) {
+        const current = [i];
+        for (let j = 1; j <= s.length; j++) {
+            current[j] = Math.min(current[j - 1] + 1, previous[j] + 1, previous[j - 1] + (s[j - 1] === t[i - 1] ? 0 : 1));
+        }
+        previous = current;
+    }
+    return previous[s.length];
+}
+/** 1 for identical, 0 for nothing in common. */
+export function similarity(a, b) {
+    if (a === "" && b === "")
+        return 1;
+    const longest = Math.max(a.length, b.length);
+    if (longest === 0)
+        return 1;
+    return Math.max(0, 1 - editDistance(a, b) / longest);
+}
+export const FIELD_NORMALIZERS = ["none", "name", "email", "phone", "company"];
+/** Apply one field normalizer, reusing the contact rules. */
+export function normalizeField(value, how) {
+    const text = value.trim();
+    switch (how) {
+        case "name":
+            return normalizeContact({ name: text }).nameKey;
+        case "email":
+            return normalizeContact({ email: text }).email;
+        case "phone":
+            return normalizeContact({ phone: text }).phone;
+        case "company":
+            return normalizeContact({ company: text }).company;
+        default:
+            return text.toLowerCase();
+    }
+}
+/**
+ * How much of a value a fuzzy comparison looks at.
+ *
+ * Edit distance is quadratic in the length, so the real cost of a linkage
+ * run is comparisons times length squared — and the comparison cap alone
+ * bounds the wrong quantity. Thirty records against thirty, with
+ * three-thousand-character values, is 900 comparisons (far under any
+ * sensible cap) and took forty-five seconds.
+ *
+ * The fields this tool exists to match — names, companies, addresses — are
+ * nowhere near this long, and two values that agree for 256 characters and
+ * differ afterwards are the same value for matching purposes.
+ */
+const MAX_COMPARE_CHARS = 256;
+const bounded = (value) => value.length <= MAX_COMPARE_CHARS ? value : value.slice(0, MAX_COMPARE_CHARS);
+/**
+ * Match records between two lists that share no key.
+ *
+ * Every accepted pair carries its evidence — which field matched and how
+ * well — because "these are the same customer" is a claim somebody will have
+ * to defend, and a bare score cannot be defended. Pairs between the review
+ * floor and the accept floor are returned separately rather than guessed at.
+ *
+ * Matching is greedy over the scored pairs, and one record is used once: a
+ * record matched to two others is not a match, it is a question.
+ */
+export function linkRecords(left, right, rules, options = {}) {
+    if (rules.length === 0)
+        throw new Error("linkage needs at least one rule");
+    const totalWeight = rules.reduce((s, r) => s + r.weight, 0);
+    if (totalWeight <= 0)
+        throw new Error("the rule weights sum to zero, so every score would be zero");
+    const accept = options.accept ?? 0.85;
+    const review = options.review ?? 0.6;
+    const maxComparisons = options.maxComparisons ?? 5_000_000;
+    if (left.length * right.length > maxComparisons) {
+        throw new Error(`${left.length} x ${right.length} is ${left.length * right.length} comparisons, over the ${maxComparisons} limit — narrow the inputs first`);
+    }
+    const scored = [];
+    for (const [i, a] of left.entries()) {
+        for (const [j, b] of right.entries()) {
+            const evidence = [];
+            let score = 0;
+            for (const rule of rules) {
+                const x = bounded(normalizeField(a[rule.field] ?? "", rule.normalize));
+                const y = bounded(normalizeField(b[rule.field] ?? "", rule.normalize));
+                if (x === "" || y === "")
+                    continue;
+                const sim = rule.compare === "exact" ? (x === y ? 1 : 0) : similarity(x, y);
+                const counted = rule.compare === "fuzzy" && sim < (rule.threshold ?? 0.8) ? 0 : sim;
+                if (counted > 0) {
+                    score += counted * rule.weight;
+                    evidence.push({
+                        field: rule.field,
+                        similarity: Number(sim.toFixed(4)),
+                        weight: rule.weight,
+                    });
+                }
+            }
+            const normalized = score / totalWeight;
+            if (normalized >= review) {
+                scored.push({
+                    leftIndex: i,
+                    rightIndex: j,
+                    score: Number(normalized.toFixed(4)),
+                    evidence,
+                });
+            }
+        }
+    }
+    scored.sort((p, q) => q.score - p.score || p.leftIndex - q.leftIndex || p.rightIndex - q.rightIndex);
+    const usedLeft = new Set();
+    const usedRight = new Set();
+    const matched = [];
+    const reviewPairs = [];
+    for (const pair of scored) {
+        if (usedLeft.has(pair.leftIndex) || usedRight.has(pair.rightIndex))
+            continue;
+        if (pair.score >= accept) {
+            matched.push(pair);
+            usedLeft.add(pair.leftIndex);
+            usedRight.add(pair.rightIndex);
+        }
+        else {
+            reviewPairs.push(pair);
+        }
+    }
+    return {
+        matched,
+        review: reviewPairs,
+        unmatchedLeft: left.map((_, i) => i).filter((i) => !usedLeft.has(i)),
+        unmatchedRight: right.map((_, i) => i).filter((i) => !usedRight.has(i)),
+        comparisons: left.length * right.length,
+    };
+}

package/dist/lib/reshape.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Reshaping and splitting tables.
+ *
+ * Wide-to-long and long-to-wide are the two transforms SQL cannot express
+ * without already knowing the distinct values, and they are exactly what
+ * stands between an export and a usable dataset. Sharding is here because a
+ * file too large to process in one pass is a practical problem with a
+ * mechanical answer.
+ */
+export type Row = Readonly<Record<string, string>>;
+/**
+ * Wide to long: one row per (identifier, variable, value).
+ *
+ * A cell that is empty is dropped by default. A wide export usually has a
+ * column per period and most cells empty, and keeping them turns a small
+ * table into a huge one whose rows say nothing.
+ */
+export declare function toLong(rows: ReadonlyArray<Row>, idColumns: ReadonlyArray<string>, valueColumns: ReadonlyArray<string>, options?: {
+    variableName?: string;
+    valueName?: string;
+    keepEmpty?: boolean;
+}): Row[];
+export type WideResult = {
+    readonly rows: ReadonlyArray<Row>;
+    readonly columns: ReadonlyArray<string>;
+    /** (identifier, variable) pairs that appeared more than once. */
+    readonly collisions: ReadonlyArray<{
+        readonly key: string;
+        readonly variable: string;
+        readonly count: number;
+    }>;
+};
+export declare function toWide(rows: ReadonlyArray<Row>, idColumns: ReadonlyArray<string>, variableColumn: string, valueColumn: string, options?: {
+    fill?: string;
+    maxColumns?: number;
+}): WideResult;
+export type Shard = {
+    readonly index: number;
+    readonly rows: number;
+    readonly bytes: number;
+    readonly firstRow: number;
+};
+/**
+ * Split rows into shards, each carrying the header.
+ *
+ * Every shard is independently readable. A split that put the header only on
+ * the first piece produces one usable file and N-1 that need it grafted back
+ * on, which is how a column ends up shifted.
+ */
+export declare function shardRows(header: ReadonlyArray<string>, rows: ReadonlyArray<ReadonlyArray<string>>, options: {
+    maxRows?: number;
+    maxBytes?: number;
+}): {
+    shards: Shard[];
+    bodies: string[];
+};
+export type FixedField = {
+    readonly name: string;
+    /** 1-based, inclusive, as a copybook or layout document states them. */
+    readonly start: number;
+    readonly length: number;
+    readonly trim?: boolean;
+};
+/**
+ * Parse a fixed-width extract.
+ *
+ * Positions are 1-based and inclusive because that is how every layout
+ * document, copybook and mainframe spec states them; converting in your head
+ * is how a field ends up one character off down the whole file.
+ */
+export declare function parseFixedWidth(text: string, fields: ReadonlyArray<FixedField>, options?: {
+    skipLines?: number;
+}): {
+    rows: Row[];
+    shortLines: Array<{
+        line: number;
+        length: number;
+    }>;
+};

package/dist/lib/reshape.js

@@ -0,0 +1,176 @@
+/**
+ * Reshaping and splitting tables.
+ *
+ * Wide-to-long and long-to-wide are the two transforms SQL cannot express
+ * without already knowing the distinct values, and they are exactly what
+ * stands between an export and a usable dataset. Sharding is here because a
+ * file too large to process in one pass is a practical problem with a
+ * mechanical answer.
+ */
+/**
+ * Wide to long: one row per (identifier, variable, value).
+ *
+ * A cell that is empty is dropped by default. A wide export usually has a
+ * column per period and most cells empty, and keeping them turns a small
+ * table into a huge one whose rows say nothing.
+ */
+export function toLong(rows, idColumns, valueColumns, options = {}) {
+    const variableName = options.variableName ?? "variable";
+    const valueName = options.valueName ?? "value";
+    if (idColumns.includes(variableName) || idColumns.includes(valueName)) {
+        throw new Error(`"${variableName}" and "${valueName}" name the output columns, so they cannot also be identifier columns — rename one`);
+    }
+    const out = [];
+    for (const row of rows) {
+        const identity = {};
+        for (const column of idColumns)
+            identity[column] = row[column] ?? "";
+        for (const column of valueColumns) {
+            const value = row[column] ?? "";
+            if (value === "" && options.keepEmpty !== true)
+                continue;
+            out.push({ ...identity, [variableName]: column, [valueName]: value });
+        }
+    }
+    return out;
+}
+/**
+ * Long to wide: one column per distinct variable.
+ *
+ * A repeated (identifier, variable) pair is a collision — the shape does not
+ * say which value wins. The first is kept and every collision is reported,
+ * because silently taking the last would make a pivot that looks complete
+ * and is wrong in a way nothing downstream can detect.
+ */
+/**
+ * How many distinct values may become columns.
+ *
+ * Pivoting on a high-cardinality column is almost always a mistake — an id
+ * or a timestamp produces a column per row — and the result is a table
+ * nobody can read rather than an error anybody notices.
+ */
+const MAX_PIVOT_COLUMNS = 2_048;
+export function toWide(rows, idColumns, variableColumn, valueColumn, options = {}) {
+    const fill = options.fill ?? "";
+    const maxColumns = options.maxColumns ?? MAX_PIVOT_COLUMNS;
+    const byKey = new Map();
+    const variables = new Set();
+    const collisions = [];
+    const counts = new Map();
+    for (const row of rows) {
+        const identity = {};
+        for (const column of idColumns)
+            identity[column] = row[column] ?? "";
+        const key = JSON.stringify(idColumns.map((c) => identity[c]));
+        const variable = row[variableColumn] ?? "";
+        if (variable === "")
+            continue;
+        variables.add(variable);
+        const pair = `${key}\t${variable}`;
+        counts.set(pair, (counts.get(pair) ?? 0) + 1);
+        let bucket = byKey.get(key);
+        if (bucket === undefined) {
+            bucket = { identity, values: new Map() };
+            byKey.set(key, bucket);
+        }
+        if (!bucket.values.has(variable))
+            bucket.values.set(variable, row[valueColumn] ?? "");
+    }
+    if (variables.size > maxColumns) {
+        throw new Error(`"${variableColumn}" has ${variables.size} distinct values, so pivoting on it would make ${variables.size} columns (limit ${maxColumns}) — that is usually an id or a timestamp rather than a category`);
+    }
+    for (const [pair, count] of counts) {
+        if (count <= 1)
+            continue;
+        const [key = "", variable = ""] = pair.split("\t");
+        collisions.push({ key, variable, count });
+    }
+    const ordered = [...variables].sort();
+    const out = [];
+    for (const bucket of byKey.values()) {
+        const row = { ...bucket.identity };
+        for (const variable of ordered)
+            row[variable] = bucket.values.get(variable) ?? fill;
+        out.push(row);
+    }
+    return { rows: out, columns: [...idColumns, ...ordered], collisions };
+}
+/**
+ * Split rows into shards, each carrying the header.
+ *
+ * Every shard is independently readable. A split that put the header only on
+ * the first piece produces one usable file and N-1 that need it grafted back
+ * on, which is how a column ends up shifted.
+ */
+export function shardRows(header, rows, options) {
+    const maxRows = options.maxRows ?? Number.POSITIVE_INFINITY;
+    const maxBytes = options.maxBytes ?? Number.POSITIVE_INFINITY;
+    if (maxRows === Number.POSITIVE_INFINITY && maxBytes === Number.POSITIVE_INFINITY) {
+        throw new Error("give maxRows or maxBytes, or every row lands in one shard");
+    }
+    const quote = (cell) => /[",\n\r]/.test(cell) ? `"${cell.split('"').join('""')}"` : cell;
+    const line = (cells) => cells.map(quote).join(",");
+    const headerLine = `${line(header)}\n`;
+    const shards = [];
+    const bodies = [];
+    let current = [];
+    let bytes = Buffer.byteLength(headerLine);
+    let firstRow = 1;
+    const flush = () => {
+        if (current.length === 0)
+            return;
+        shards.push({ index: shards.length, rows: current.length, bytes, firstRow });
+        bodies.push(headerLine + current.join(""));
+        current = [];
+        bytes = Buffer.byteLength(headerLine);
+    };
+    for (const [i, row] of rows.entries()) {
+        const text = `${line(row)}\n`;
+        const size = Buffer.byteLength(text);
+        if (current.length > 0 && (current.length + 1 > maxRows || bytes + size > maxBytes)) {
+            flush();
+            firstRow = i + 1;
+        }
+        current.push(text);
+        bytes += size;
+    }
+    flush();
+    return { shards, bodies };
+}
+/**
+ * Parse a fixed-width extract.
+ *
+ * Positions are 1-based and inclusive because that is how every layout
+ * document, copybook and mainframe spec states them; converting in your head
+ * is how a field ends up one character off down the whole file.
+ */
+export function parseFixedWidth(text, fields, options = {}) {
+    if (fields.length === 0)
+        throw new Error("a fixed-width layout needs at least one field");
+    for (const field of fields) {
+        if (field.start < 1)
+            throw new Error(`field "${field.name}" starts at ${field.start}; positions are 1-based`);
+        if (field.length < 1)
+            throw new Error(`field "${field.name}" has length ${field.length}`);
+    }
+    const needed = Math.max(...fields.map((f) => f.start + f.length - 1));
+    const rows = [];
+    const shortLines = [];
+    const lines = text.split(/\r?\n/).slice(options.skipLines ?? 0);
+    for (const [i, raw] of lines.entries()) {
+        if (raw.trim() === "")
+            continue;
+        // A short line is reported rather than padded: silently padding turns a
+        // truncated record into one with empty trailing fields, which reads as
+        // real data.
+        if (raw.length < needed)
+            shortLines.push({ line: i + 1 + (options.skipLines ?? 0), length: raw.length });
+        const row = {};
+        for (const field of fields) {
+            const value = raw.slice(field.start - 1, field.start - 1 + field.length);
+            row[field.name] = field.trim === false ? value : value.trim();
+        }
+        rows.push(row);
+    }
+    return { rows, shortLines };
+}

package/dist/paths.d.ts

@@ -0,0 +1,42 @@
+import { CrewhausError } from "@crewhaus/errors";
+/** Raised when a caller-supplied path resolves outside the workspace root. */
+export declare class ToolPermissionError extends CrewhausError {
+    readonly name = "ToolPermissionError";
+    readonly toolName: string;
+    readonly path: string;
+    constructor(toolName: string, attemptedPath: string);
+}
+/**
+ * A validated path, in both forms the tools need.
+ *
+ * `real` has every in-workspace symlink already resolved and is what I/O
+ * should use — operating on it closes most of the check-to-use window.
+ * `abs` is the lexical resolution, which is what a tool must use when the
+ * question is about the link ITSELF (`Stat` reporting a symlink target,
+ * `RemovePath` deleting a link rather than its destination).
+ */
+export type SafePath = {
+    readonly abs: string;
+    readonly real: string;
+    /** The path as the caller wrote it, for echoing back in results. */
+    readonly given: string;
+    /** Slash-separated path relative to the workspace root. */
+    readonly rel: string;
+};
+/** The workspace root, resolved. Every path is contained within it. */
+export declare function workspaceRoot(): string;
+export declare function resolveSafe(toolName: string, rel: string, root?: string): SafePath;
+/** A path relative to the workspace root, with `/` separators on every OS. */
+export declare function toPosix(p: string): string;
+/**
+ * True when `candidate` is `parent` or lives underneath it. Both must already
+ * be absolute. Used to keep an archive entry, or a symlink inside an
+ * extracted tree, from reaching outside its destination.
+ */
+export declare function isInside(parent: string, candidate: string): boolean;
+/**
+ * Reject an archive member name that would escape its destination: an
+ * absolute path, a Windows drive prefix, or any `..` segment. Checked on the
+ * NAME, before extraction, because after extraction the damage is done.
+ */
+export declare function archiveEntryEscapes(name: string): boolean;