@crewhaus/tool-data 0.7.0

package/dist/lib/csv.d.ts

@@ -0,0 +1,80 @@
+/**
+ * CSV reading and writing to RFC 4180, plus the two departures from it that
+ * real files require: a configurable delimiter (tab, semicolon, pipe) and
+ * tolerance of bare LF line endings as well as CRLF.
+ *
+ * What is handled: quoted fields, delimiters and newlines inside quotes,
+ * doubled quotes as an escaped quote, a final row with no trailing newline,
+ * a row that is a single quoted empty field (`""`, which is a row and not a
+ * blank line), and ragged rows (reported, not silently padded, unless
+ * asked).
+ *
+ * What is not: a byte-order mark is stripped but other encodings are the
+ * caller's problem, and there is no support for backslash escapes, which
+ * RFC 4180 does not define.
+ */
+export type CsvParseOptions = {
+    delimiter: string;
+    quote: string;
+    /** Drop lines that are entirely empty, rather than yielding a one-empty-field row. */
+    skipEmptyLines: boolean;
+    /** Trim surrounding whitespace from every unquoted field. Quoted fields are never trimmed. */
+    trim: boolean;
+    /** Stop after this many rows (header included) and report the truncation. */
+    maxRows: number;
+};
+export type CsvParseResult = {
+    rows: string[][];
+    truncated: boolean;
+};
+export declare class CsvError extends Error {
+    readonly line: number;
+    constructor(message: string, line: number);
+}
+/** Split CSV text into rows of raw string fields. Throws `CsvError` on an unterminated quote. */
+export declare function parseCsv(text: string, options: CsvParseOptions): CsvParseResult;
+/**
+ * Best-effort scalar typing of a CSV cell: integers and decimals become
+ * numbers, `true`/`false` (any case) become booleans, an empty cell and the
+ * configured null tokens become null, everything else stays a string.
+ *
+ * Leading zeros (`007`), leading `+`, and values outside the safe integer
+ * range stay strings, because turning a zip code or an account number into a
+ * number loses data.
+ */
+export declare function inferScalar(raw: string, nullTokens: ReadonlyArray<string>): unknown;
+/** Turn rows into records using the first row as the header. */
+export declare function rowsToRecords(rows: ReadonlyArray<ReadonlyArray<string>>, header: ReadonlyArray<string>, infer: boolean, nullTokens: ReadonlyArray<string>): {
+    records: Record<string, unknown>[];
+    ragged: number[];
+};
+/**
+ * Make header names unique and non-empty, so records never lose a column.
+ *
+ * The suffix is bumped until the name is genuinely unused, not merely until
+ * the base name's counter advances: `a, a, a_2` has to come back as
+ * `a, a_2, a_3`, because emitting `a_2` twice would collapse two columns
+ * into one when the rows are turned into records.
+ */
+export declare function normalizeHeader(raw: ReadonlyArray<string>): string[];
+export type CsvWriteOptions = {
+    delimiter: string;
+    quote: string;
+    /** "\n" or "\r\n"; RFC 4180 specifies CRLF, but LF is what most tools now emit. */
+    newline: string;
+    /** Quote every field, not only the ones that need it. */
+    quoteAll: boolean;
+    header: string[] | null;
+};
+/** Quote a single field if it contains the delimiter, a quote, or a line break. */
+export declare function escapeCsvField(value: string, options: CsvWriteOptions): string;
+/**
+ * Render a cell value. `null` and `undefined` become the empty field;
+ * objects and arrays become their compact JSON, because silently emitting
+ * `[object Object]` is how a pipeline loses data without noticing.
+ */
+export declare function cellToString(value: unknown): string;
+/** Serialize rows of already-stringified cells. */
+export declare function writeCsvRows(rows: ReadonlyArray<ReadonlyArray<unknown>>, options: CsvWriteOptions): string;
+/** Column order for a set of records: keys in first-seen order across every record. */
+export declare function unionKeys(records: ReadonlyArray<Record<string, unknown>>): string[];

package/dist/lib/csv.js

@@ -0,0 +1,240 @@
+/**
+ * CSV reading and writing to RFC 4180, plus the two departures from it that
+ * real files require: a configurable delimiter (tab, semicolon, pipe) and
+ * tolerance of bare LF line endings as well as CRLF.
+ *
+ * What is handled: quoted fields, delimiters and newlines inside quotes,
+ * doubled quotes as an escaped quote, a final row with no trailing newline,
+ * a row that is a single quoted empty field (`""`, which is a row and not a
+ * blank line), and ragged rows (reported, not silently padded, unless
+ * asked).
+ *
+ * What is not: a byte-order mark is stripped but other encodings are the
+ * caller's problem, and there is no support for backslash escapes, which
+ * RFC 4180 does not define.
+ */
+export class CsvError extends Error {
+    line;
+    constructor(message, line) {
+        super(message);
+        this.line = line;
+    }
+}
+/** Split CSV text into rows of raw string fields. Throws `CsvError` on an unterminated quote. */
+export function parseCsv(text, options) {
+    const { delimiter, quote } = options;
+    if (delimiter.length !== 1)
+        throw new CsvError("delimiter must be a single character", 0);
+    if (quote.length !== 1)
+        throw new CsvError("quote must be a single character", 0);
+    if (delimiter === quote)
+        throw new CsvError("delimiter and quote must differ", 0);
+    const src = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+    const rows = [];
+    let row = [];
+    let field = "";
+    let quoted = false;
+    let fieldWasQuoted = false;
+    // Whether anything in the current row was written with quotes. A row that
+    // is a single *quoted* empty field (`""`) is a real one-field row and must
+    // not be mistaken for a blank line.
+    let rowHadQuote = false;
+    let line = 1;
+    // Where the currently-open quote started, so an unterminated one points at
+    // the opening quote rather than at the end of the file.
+    let quoteStartLine = 1;
+    let truncated = false;
+    const endField = () => {
+        row.push(options.trim && !fieldWasQuoted ? field.trim() : field);
+        field = "";
+        fieldWasQuoted = false;
+    };
+    const endRow = () => {
+        endField();
+        const blank = row.length === 1 && row[0] === "" && !rowHadQuote;
+        if (!(options.skipEmptyLines && blank))
+            rows.push(row);
+        row = [];
+        rowHadQuote = false;
+        if (rows.length >= options.maxRows) {
+            truncated = true;
+            return false;
+        }
+        return true;
+    };
+    for (let i = 0; i < src.length; i++) {
+        const c = src[i];
+        if (quoted) {
+            if (c === quote) {
+                if (src[i + 1] === quote) {
+                    field += quote;
+                    i += 1;
+                    continue;
+                }
+                quoted = false;
+                continue;
+            }
+            if (c === "\n")
+                line += 1;
+            field += c;
+            continue;
+        }
+        if (c === quote && field === "" && !fieldWasQuoted) {
+            quoted = true;
+            fieldWasQuoted = true;
+            rowHadQuote = true;
+            quoteStartLine = line;
+            continue;
+        }
+        if (c === delimiter) {
+            endField();
+            continue;
+        }
+        if (c === "\r") {
+            if (src[i + 1] === "\n")
+                i += 1;
+            line += 1;
+            if (!endRow())
+                return { rows, truncated };
+            continue;
+        }
+        if (c === "\n") {
+            line += 1;
+            if (!endRow())
+                return { rows, truncated };
+            continue;
+        }
+        field += c;
+    }
+    if (quoted) {
+        throw new CsvError(`unterminated quoted field — the quote opened on line ${quoteStartLine} is never closed`, quoteStartLine);
+    }
+    // A trailing newline should not manufacture an extra empty row — but a
+    // final `""` is a row, so the quote flag has to be part of the test.
+    if (field !== "" || row.length > 0 || fieldWasQuoted)
+        endRow();
+    return { rows, truncated };
+}
+/**
+ * Best-effort scalar typing of a CSV cell: integers and decimals become
+ * numbers, `true`/`false` (any case) become booleans, an empty cell and the
+ * configured null tokens become null, everything else stays a string.
+ *
+ * Leading zeros (`007`), leading `+`, and values outside the safe integer
+ * range stay strings, because turning a zip code or an account number into a
+ * number loses data.
+ */
+export function inferScalar(raw, nullTokens) {
+    const s = raw.trim();
+    if (s === "")
+        return null;
+    if (nullTokens.includes(s))
+        return null;
+    const lower = s.toLowerCase();
+    if (lower === "true")
+        return true;
+    if (lower === "false")
+        return false;
+    if (/^-?(0|[1-9][0-9]*)$/.test(s)) {
+        const n = Number(s);
+        return Number.isSafeInteger(n) ? n : raw;
+    }
+    if (/^-?(0|[1-9][0-9]*)\.[0-9]+$/.test(s))
+        return Number(s);
+    return raw;
+}
+/** Turn rows into records using the first row as the header. */
+export function rowsToRecords(rows, header, infer, nullTokens) {
+    const records = [];
+    const ragged = [];
+    rows.forEach((row, i) => {
+        if (row.length !== header.length)
+            ragged.push(i);
+        const rec = {};
+        header.forEach((name, j) => {
+            const cell = row[j];
+            if (cell === undefined) {
+                rec[name] = null;
+                return;
+            }
+            rec[name] = infer ? inferScalar(cell, nullTokens) : cell;
+        });
+        records.push(rec);
+    });
+    return { records, ragged };
+}
+/**
+ * Make header names unique and non-empty, so records never lose a column.
+ *
+ * The suffix is bumped until the name is genuinely unused, not merely until
+ * the base name's counter advances: `a, a, a_2` has to come back as
+ * `a, a_2, a_3`, because emitting `a_2` twice would collapse two columns
+ * into one when the rows are turned into records.
+ */
+export function normalizeHeader(raw) {
+    const counts = new Map();
+    const taken = new Set();
+    return raw.map((name, i) => {
+        const base = name.trim() === "" ? `column${i + 1}` : name.trim();
+        let count = counts.get(base) ?? 0;
+        let candidate = count === 0 ? base : `${base}_${count + 1}`;
+        while (taken.has(candidate)) {
+            count += 1;
+            candidate = `${base}_${count + 1}`;
+        }
+        counts.set(base, count + 1);
+        taken.add(candidate);
+        return candidate;
+    });
+}
+/** Quote a single field if it contains the delimiter, a quote, or a line break. */
+export function escapeCsvField(value, options) {
+    const needs = options.quoteAll ||
+        value.includes(options.delimiter) ||
+        value.includes(options.quote) ||
+        value.includes("\n") ||
+        value.includes("\r");
+    if (!needs)
+        return value;
+    const q = options.quote;
+    return q + value.split(q).join(q + q) + q;
+}
+/**
+ * Render a cell value. `null` and `undefined` become the empty field;
+ * objects and arrays become their compact JSON, because silently emitting
+ * `[object Object]` is how a pipeline loses data without noticing.
+ */
+export function cellToString(value) {
+    if (value === null || value === undefined)
+        return "";
+    if (typeof value === "string")
+        return value;
+    if (typeof value === "number" || typeof value === "boolean")
+        return String(value);
+    return JSON.stringify(value);
+}
+/** Serialize rows of already-stringified cells. */
+export function writeCsvRows(rows, options) {
+    const lines = [];
+    if (options.header !== null) {
+        lines.push(options.header.map((h) => escapeCsvField(h, options)).join(options.delimiter));
+    }
+    for (const row of rows) {
+        lines.push(row.map((cell) => escapeCsvField(cellToString(cell), options)).join(options.delimiter));
+    }
+    return lines.join(options.newline);
+}
+/** Column order for a set of records: keys in first-seen order across every record. */
+export function unionKeys(records) {
+    const out = [];
+    const seen = new Set();
+    for (const rec of records) {
+        for (const k of Object.keys(rec)) {
+            if (!seen.has(k)) {
+                seen.add(k);
+                out.push(k);
+            }
+        }
+    }
+    return out;
+}

package/dist/lib/diff.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Deep structural diff of two JSON values.
+ *
+ * Objects are compared key by key; arrays are compared index by index, which
+ * means an insertion at the front reports every later index as changed. That
+ * is the honest answer for positional data (a config array, a tuple) and the
+ * wrong one for a list of records — for records, key the arrays first with
+ * `keyArraysBy`, which matches elements by a field instead of by position.
+ */
+export type DiffKind = "added" | "removed" | "changed";
+export type DiffEntry = {
+    path: string;
+    kind: DiffKind;
+    /** The type on each side, so a `1` -> `"1"` change is visible. */
+    from?: unknown;
+    to?: unknown;
+    fromType?: string;
+    toType?: string;
+};
+export type DiffOptions = {
+    /**
+     * Match array elements by the value of this field instead of by index.
+     * The keying is all-or-nothing per array: if *either* side has an element
+     * without the field, with a non-scalar value for it, or with a value that
+     * repeats, that whole array is compared positionally instead. Nested
+     * arrays are considered independently.
+     */
+    keyArraysBy?: string;
+    /** Stop descending past this depth; deeper differences report as one `changed`. */
+    maxDepth: number;
+    /** Stop after this many entries, so a total rewrite cannot fill a context window. */
+    maxEntries: number;
+};
+export type DiffResult = {
+    entries: DiffEntry[];
+    truncated: boolean;
+    counts: {
+        added: number;
+        removed: number;
+        changed: number;
+    };
+};
+/** Compare two values, returning a flat list of differences by JSON Pointer path. */
+export declare function deepDiff(a: unknown, b: unknown, options: DiffOptions): DiffResult;

package/dist/lib/diff.js

@@ -0,0 +1,166 @@
+/**
+ * Deep structural diff of two JSON values.
+ *
+ * Objects are compared key by key; arrays are compared index by index, which
+ * means an insertion at the front reports every later index as changed. That
+ * is the honest answer for positional data (a config array, a tuple) and the
+ * wrong one for a list of records — for records, key the arrays first with
+ * `keyArraysBy`, which matches elements by a field instead of by position.
+ */
+import { deepEqual, isPlainObject, typeOf } from "./json";
+import { formatPointer } from "./patch";
+/** Compare two values, returning a flat list of differences by JSON Pointer path. */
+export function deepDiff(a, b, options) {
+    const entries = [];
+    let truncated = false;
+    const push = (entry) => {
+        if (entries.length >= options.maxEntries) {
+            truncated = true;
+            return;
+        }
+        entries.push(entry);
+    };
+    const recurse = (left, right, path, depth) => {
+        if (truncated)
+            return;
+        // Equality first, then the budget: returning early on a full budget
+        // *before* knowing there is a difference would leave `truncated` false
+        // while differences went unreported, which is exactly the silence this
+        // tool exists to avoid.
+        if (deepEqual(left, right))
+            return;
+        if (entries.length >= options.maxEntries) {
+            truncated = true;
+            return;
+        }
+        const pointer = formatPointer(path) || "/";
+        if (depth >= options.maxDepth) {
+            push({
+                path: pointer,
+                kind: "changed",
+                from: left,
+                to: right,
+                fromType: typeOf(left),
+                toType: typeOf(right),
+            });
+            return;
+        }
+        if (isPlainObject(left) && isPlainObject(right)) {
+            for (const key of Object.keys(left)) {
+                if (!Object.hasOwn(right, key)) {
+                    push({
+                        path: formatPointer([...path, key]),
+                        kind: "removed",
+                        from: left[key],
+                        fromType: typeOf(left[key]),
+                    });
+                    continue;
+                }
+                recurse(left[key], right[key], [...path, key], depth + 1);
+            }
+            for (const key of Object.keys(right)) {
+                if (!Object.hasOwn(left, key)) {
+                    push({
+                        path: formatPointer([...path, key]),
+                        kind: "added",
+                        to: right[key],
+                        toType: typeOf(right[key]),
+                    });
+                }
+            }
+            return;
+        }
+        if (Array.isArray(left) && Array.isArray(right)) {
+            const key = options.keyArraysBy;
+            if (key !== undefined && canKey(left, key) && canKey(right, key)) {
+                diffKeyedArrays(left, right, key, path, depth, recurse, push);
+                return;
+            }
+            const shared = Math.min(left.length, right.length);
+            for (let i = 0; i < shared; i++) {
+                recurse(left[i], right[i], [...path, String(i)], depth + 1);
+            }
+            for (let i = shared; i < left.length; i++) {
+                push({
+                    path: formatPointer([...path, String(i)]),
+                    kind: "removed",
+                    from: left[i],
+                    fromType: typeOf(left[i]),
+                });
+            }
+            for (let i = shared; i < right.length; i++) {
+                push({
+                    path: formatPointer([...path, String(i)]),
+                    kind: "added",
+                    to: right[i],
+                    toType: typeOf(right[i]),
+                });
+            }
+            return;
+        }
+        push({
+            path: pointer,
+            kind: "changed",
+            from: left,
+            to: right,
+            fromType: typeOf(left),
+            toType: typeOf(right),
+        });
+    };
+    recurse(a, b, [], 0);
+    const counts = { added: 0, removed: 0, changed: 0 };
+    for (const e of entries)
+        counts[e.kind] += 1;
+    return { entries, truncated, counts };
+}
+/** True when every element is an object carrying a distinct, scalar value for `key`. */
+function canKey(arr, key) {
+    const seen = new Set();
+    for (const el of arr) {
+        if (!isPlainObject(el))
+            return false;
+        const v = el[key];
+        if (v === undefined || v === null || typeof v === "object")
+            return false;
+        const s = String(v);
+        if (seen.has(s))
+            return false;
+        seen.add(s);
+    }
+    return true;
+}
+function diffKeyedArrays(left, right, key, path, depth, recurse, push) {
+    const index = (arr) => {
+        const m = new Map();
+        arr.forEach((el, i) => {
+            const v = el[key];
+            m.set(String(v), { i, value: el });
+        });
+        return m;
+    };
+    const l = index(left);
+    const r = index(right);
+    for (const [k, entry] of l) {
+        const other = r.get(k);
+        if (other === undefined) {
+            push({
+                path: formatPointer([...path, String(entry.i)]),
+                kind: "removed",
+                from: entry.value,
+                fromType: typeOf(entry.value),
+            });
+            continue;
+        }
+        recurse(entry.value, other.value, [...path, String(other.i)], depth + 1);
+    }
+    for (const [k, entry] of r) {
+        if (!l.has(k)) {
+            push({
+                path: formatPointer([...path, String(entry.i)]),
+                kind: "added",
+                to: entry.value,
+                toType: typeOf(entry.value),
+            });
+        }
+    }
+}

package/dist/lib/json.d.ts

@@ -0,0 +1,65 @@
+/**
+ * The JSON primitives everything else in this package is built on: a value
+ * model, structural equality, a stable (key-sorted) serializer, and a
+ * content hash.
+ *
+ * Stability is the point. Two harness runs that produce the same data must
+ * produce the same bytes, or every downstream diff is noise.
+ */
+export type JsonValue = null | boolean | number | string | JsonValue[] | {
+    [k: string]: JsonValue;
+};
+/** A plain object, as distinct from an array or a primitive. */
+export declare function isPlainObject(value: unknown): value is Record<string, unknown>;
+/** Structural equality over JSON values. Key order is irrelevant; NaN never appears in JSON. */
+export declare function deepEqual(a: unknown, b: unknown): boolean;
+/** A structured clone that only has to handle JSON shapes, so no cycles exist. */
+export declare function deepClone<T>(value: T): T;
+/**
+ * Serialize with object keys in sorted order at every depth, so that two
+ * equal values always produce identical bytes. Array order is data and is
+ * never touched.
+ *
+ * One caveat worth knowing rather than discovering: `JSON.stringify` emits
+ * an object's integer-like keys first, in ascending numeric order, whatever
+ * order they sit in. So `{"10":a,"2":b,"x":c}` serializes as `2, 10, x`, not
+ * the code-unit order `10, 2, x` that `sortKeysDeep` puts them in. The
+ * result is still canonical — two equal values still produce identical bytes
+ * — but it is not literally sorted for those keys.
+ *
+ * `indent` of 0 gives the compact form.
+ */
+export declare function canonicalStringify(value: unknown, indent?: number): string;
+/** Recursively rebuild objects with their keys in code-unit sorted order. */
+export declare function sortKeysDeep(value: unknown): unknown;
+/**
+ * FNV-1a, 64 bits, over the UTF-8 bytes of `text` — the algorithm exactly as
+ * Fowler/Noll/Vo define it, so the reference vectors hold: `""` hashes to
+ * `cbf29ce484222325`, `"a"` to `af63dc4c8601ec8c` and `"foobar"` to
+ * `85944171f73967e8`. `lib.test.ts` pins all three.
+ *
+ * It is a content fingerprint, not a cryptographic digest — never use it
+ * where collision resistance against an adversary matters.
+ */
+export declare function fnv1a64(text: string): string;
+/**
+ * A 64-bit FNV-1a hash of the canonical serialization, rendered as 16 hex
+ * characters. Two values that are `deepEqual` always hash the same, because
+ * the serialization sorts keys first.
+ *
+ * Use it as a fingerprint a caller can compare or record. Do not use it as
+ * an identity key inside this package: 64 bits collide, and the canonical
+ * string it is computed from is right there and is exact, which is what
+ * `totalCompare`, `aggregate`, `joinRecords` and `dedupeRecords` key on.
+ */
+export declare function stableHash(value: unknown): string;
+/** Parse JSON, converting the engine's error into a caller-readable message. */
+export declare function parseJson(text: string): {
+    ok: true;
+    value: unknown;
+} | {
+    ok: false;
+    error: string;
+};
+/** A short, human-readable type name used in diffs and error messages. */
+export declare function typeOf(value: unknown): string;