@crewhaus/tool-data 0.7.0 → 0.8.0

package/dist/lib/table.d.ts

@@ -7,10 +7,20 @@
  * two rows compare equal the original order is kept, so the same input gives
  * the same output every time.
  */
+import { type RegexAnswers } from "@crewhaus/tool-schema";
 export type Record_ = Record<string, unknown>;
-/** Read a dotted path out of a record. Returns undefined for any miss. */
+/**
+ * Read a dotted path out of a record. Returns undefined for any miss. Only
+ * the record's own fields are read: `constructor` on a record that has no
+ * such field is a miss, not `Object.prototype.constructor`.
+ */
 export declare function getPath(record: unknown, path: string): unknown;
-/** Write a dotted path into a record, creating intermediate objects. */
+/**
+ * Write a dotted path into a record, creating intermediate objects. Every
+ * segment is an own field of the object it names — descent never follows an
+ * inherited member, so `__proto__.x` writes a field called `__proto__`, not
+ * into `Object.prototype`.
+ */
 export declare function setPath(target: Record_, path: string, value: unknown): void;
 export type Comparison = "eq" | "ne" | "lt" | "lte" | "gt" | "gte" | "contains" | "startsWith" | "endsWith" | "in" | "matches" | "exists" | "empty";
 export type Condition = {
@@ -26,8 +36,21 @@ export type Predicate = {
 };
 export declare class PredicateError extends Error {
 }
-/** Evaluate one condition against a record. */
-export declare function testCondition(record: unknown, cond: Condition): boolean;
+/** A predicate's answer for one record: a verdict, or undetermined (a pattern had no answer). */
+export type PredicateVerdict = boolean | "undetermined";
+/**
+ * Ask `regex` about every `matches` condition in `predicate` for every
+ * record, so one resolve in the regex worker answers them all before
+ * {@link testPredicate} reads them.
+ */
+export declare function askPredicatePatterns(records: ReadonlyArray<unknown>, predicate: Predicate, regex: RegexAnswers): void;
+/**
+ * Evaluate one condition against a record. A `matches` pattern is never run
+ * here: its answer comes from `regex`, resolved in the worker by the tool
+ * (or, for a library caller with none, from the bounded fallback on this
+ * thread), and one with no answer is `"undetermined"`, never `false`.
+ */
+export declare function testCondition(record: unknown, cond: Condition, regex?: RegexAnswers): PredicateVerdict;
 /**
  * Order two scalars, or null when they are not comparable. Numbers compare
  * numerically, strings by code unit, booleans false-before-true; a
@@ -35,8 +58,16 @@ export declare function testCondition(record: unknown, cond: Condition): boolean
  * quietly being true.
  */
 export declare function compareValues(a: unknown, b: unknown): number | null;
-/** Evaluate a whole predicate. An empty predicate matches everything. */
-export declare function testPredicate(record: unknown, predicate: Predicate): boolean;
+/**
+ * Evaluate a whole predicate. An empty predicate matches everything.
+ *
+ * Three-valued, so a condition with no answer cannot decide a record either
+ * way on its own: `all` fails on any definite miss, `none` excludes on any
+ * definite hit, `any` holds on any definite hit; otherwise a condition with
+ * no answer leaves the record `"undetermined"`. Reading it as a miss would
+ * let a `none` exclusion pass the very row it was written to stop.
+ */
+export declare function testPredicate(record: unknown, predicate: Predicate, regex?: RegexAnswers): PredicateVerdict;
 export type SortKey = {
     field: string;
     direction?: "asc" | "desc";
@@ -111,11 +142,17 @@ export type JoinOptions = {
  * how a join quietly loses rows.
  *
  * Every right-hand match is emitted, so a one-to-many join multiplies rows,
- * as SQL does. `maxRows` bounds that.
+ * as SQL does. `maxRows` bounds that — the WORK as well as the output: a row
+ * past the cap is counted, never built, so a join on a low-cardinality key
+ * (every row `status: "open"`) costs the rows it returns plus one pass over
+ * each side, not |left| x |right| merges thrown away. `totalRows` is the size
+ * the uncapped join would have had, counted arithmetically, so a caller can
+ * tell ten rows cut from eleven apart from ten cut from millions.
  */
 export declare function joinRecords(left: ReadonlyArray<Record_>, right: ReadonlyArray<Record_>, options: JoinOptions): {
     rows: Record_[];
     truncated: boolean;
+    totalRows: number;
     unmatchedLeft: number;
     unmatchedRight: number;
 };
@@ -130,7 +167,7 @@ export declare function columnsToRecords(columns: Record<string, ReadonlyArray<u
     ragged: string[];
 };
 /** Flatten nested objects into dotted keys. Arrays become `a.0`, `a.1` when expanded. */
-export declare function flattenObject(value: unknown, separator: string, expandArrays: boolean, maxDepth: number): Record<string, unknown>;
+export declare function flattenObject(value: unknown, separator: string, expandArrays: boolean, maxDepth: number, maxKeyChars?: number): Record<string, unknown>;
 /**
  * Rebuild nested objects from dotted keys. A numeric segment becomes an
  * array index when `arraysFromNumericKeys` is on and the keys under that

package/dist/lib/table.js

@@ -7,15 +7,20 @@
  * two rows compare equal the original order is kept, so the same input gives
  * the same output every time.
  */
-import { canonicalStringify, deepEqual, isPlainObject } from "./json";
-/** Read a dotted path out of a record. Returns undefined for any miss. */
+import { testPatternSync } from "@crewhaus/tool-schema";
+import { OutputLimitError, canonicalStringify, deepEqual, getOwn, isPlainObject, setOwn, } from "./json";
+/**
+ * Read a dotted path out of a record. Returns undefined for any miss. Only
+ * the record's own fields are read: `constructor` on a record that has no
+ * such field is a miss, not `Object.prototype.constructor`.
+ */
 export function getPath(record, path) {
     if (path === "")
         return record;
     let cur = record;
     for (const seg of path.split(".")) {
         if (isPlainObject(cur)) {
-            cur = cur[seg];
+            cur = getOwn(cur, seg);
             continue;
         }
         if (Array.isArray(cur) && /^\d+$/.test(seg)) {
@@ -26,28 +31,59 @@ export function getPath(record, path) {
     }
     return cur;
 }
-/** Write a dotted path into a record, creating intermediate objects. */
+/**
+ * Write a dotted path into a record, creating intermediate objects. Every
+ * segment is an own field of the object it names — descent never follows an
+ * inherited member, so `__proto__.x` writes a field called `__proto__`, not
+ * into `Object.prototype`.
+ */
 export function setPath(target, path, value) {
     const segs = path.split(".");
     let cur = target;
     for (let i = 0; i < segs.length - 1; i++) {
         const seg = segs[i];
-        const next = cur[seg];
+        const next = getOwn(cur, seg);
         if (!isPlainObject(next)) {
             const fresh = {};
-            cur[seg] = fresh;
+            setOwn(cur, seg, fresh);
             cur = fresh;
         }
         else {
             cur = next;
         }
     }
-    cur[segs[segs.length - 1]] = value;
+    setOwn(cur, segs[segs.length - 1], value);
 }
 export class PredicateError extends Error {
 }
-/** Evaluate one condition against a record. */
-export function testCondition(record, cond) {
+/**
+ * Ask `regex` about every `matches` condition in `predicate` for every
+ * record, so one resolve in the regex worker answers them all before
+ * {@link testPredicate} reads them.
+ */
+export function askPredicatePatterns(records, predicate, regex) {
+    const conditions = [
+        ...(predicate.all ?? []),
+        ...(predicate.any ?? []),
+        ...(predicate.none ?? []),
+    ].filter((c) => c.op === "matches" && typeof c.value === "string");
+    if (conditions.length === 0)
+        return;
+    for (const record of records) {
+        for (const cond of conditions) {
+            const actual = getPath(record, cond.field);
+            if (actual !== undefined && actual !== null)
+                regex.lookup(cond.value, "", String(actual));
+        }
+    }
+}
+/**
+ * Evaluate one condition against a record. A `matches` pattern is never run
+ * here: its answer comes from `regex`, resolved in the worker by the tool
+ * (or, for a library caller with none, from the bounded fallback on this
+ * thread), and one with no answer is `"undetermined"`, never `false`.
+ */
+export function testCondition(record, cond, regex) {
     const actual = getPath(record, cond.field);
     const expected = cond.value;
     switch (cond.op) {
@@ -78,14 +114,25 @@ export function testCondition(record, cond) {
             if (typeof expected !== "string") {
                 throw new PredicateError(`"matches" needs a regular-expression string for ${cond.field}`);
             }
-            let re;
             try {
-                re = new RegExp(expected);
+                // Compiling reads only the pattern; the match is what is not run here.
+                new RegExp(expected);
             }
             catch (err) {
                 throw new PredicateError(`invalid regex for ${cond.field}: ${err.message}`);
             }
-            return actual !== undefined && actual !== null && re.test(String(actual));
+            if (actual === undefined || actual === null)
+                return false;
+            const subject = String(actual);
+            const answer = regex === undefined
+                ? testPatternSync(expected, "", subject)
+                : regex.lookup(expected, "", subject);
+            if (typeof answer === "boolean")
+                return answer;
+            if (answer !== undefined && "refused" in answer) {
+                throw new PredicateError(`invalid regex for ${cond.field}: ${answer.refused}`);
+            }
+            return "undetermined";
         }
         default:
             return compareOrdered(actual, expected, cond.op);
@@ -120,20 +167,52 @@ export function compareValues(a, b) {
         return a === b ? 0 : a ? 1 : -1;
     return null;
 }
-/** Evaluate a whole predicate. An empty predicate matches everything. */
-export function testPredicate(record, predicate) {
+/**
+ * Evaluate a whole predicate. An empty predicate matches everything.
+ *
+ * Three-valued, so a condition with no answer cannot decide a record either
+ * way on its own: `all` fails on any definite miss, `none` excludes on any
+ * definite hit, `any` holds on any definite hit; otherwise a condition with
+ * no answer leaves the record `"undetermined"`. Reading it as a miss would
+ * let a `none` exclusion pass the very row it was written to stop.
+ */
+export function testPredicate(record, predicate, regex) {
     const all = predicate.all ?? [];
     const any = predicate.any ?? [];
     const none = predicate.none ?? [];
-    for (const c of all)
-        if (!testCondition(record, c))
+    let open = false;
+    for (const c of all) {
+        const v = testCondition(record, c, regex);
+        if (v === false)
             return false;
-    for (const c of none)
-        if (testCondition(record, c))
+        if (v === "undetermined")
+            open = true;
+    }
+    for (const c of none) {
+        const v = testCondition(record, c, regex);
+        if (v === true)
             return false;
-    if (any.length > 0 && !any.some((c) => testCondition(record, c)))
-        return false;
-    return true;
+        if (v === "undetermined")
+            open = true;
+    }
+    if (any.length > 0) {
+        let hit = false;
+        let anyOpen = false;
+        for (const c of any) {
+            const v = testCondition(record, c, regex);
+            if (v === true) {
+                hit = true;
+                break;
+            }
+            if (v === "undetermined")
+                anyOpen = true;
+        }
+        if (!hit && !anyOpen)
+            return false;
+        if (!hit)
+            open = true;
+    }
+    return open ? "undetermined" : true;
 }
 /**
  * Sort records by one or more keys. The sort is stable and total: values of
@@ -243,10 +322,11 @@ export function omitFields(record, fields) {
         for (let i = 0; i < segs.length - 1; i++) {
             if (!isPlainObject(cur))
                 break;
-            cur = cur[segs[i]];
+            cur = getOwn(cur, segs[i]);
         }
-        if (isPlainObject(cur))
-            delete cur[segs[segs.length - 1]];
+        const last = segs[segs.length - 1];
+        if (isPlainObject(cur) && Object.hasOwn(cur, last))
+            delete cur[last];
     }
     return out;
 }
@@ -254,7 +334,7 @@ function structuredCloneish(value) {
     const out = {};
     for (const k of Object.keys(value)) {
         const v = value[k];
-        out[k] = isPlainObject(v) ? structuredCloneish(v) : v;
+        setOwn(out, k, isPlainObject(v) ? structuredCloneish(v) : v);
     }
     return out;
 }
@@ -271,7 +351,7 @@ export function aggregate(records, groupBy, aggregations) {
     for (const rec of records) {
         const key = {};
         for (const field of groupBy)
-            key[field] = getPath(rec, field) ?? null;
+            setOwn(key, field, getPath(rec, field) ?? null);
         const id = canonicalStringify(key);
         const existing = buckets.get(id);
         if (existing === undefined)
@@ -288,37 +368,37 @@ export function aggregate(records, groupBy, aggregations) {
             const values = bucket.rows.map((r) => getPath(r, field));
             switch (agg.fn) {
                 case "count":
-                    out[agg.as] = field === "" ? bucket.rows.length : values.filter(isPresent).length;
+                    setOwn(out, agg.as, field === "" ? bucket.rows.length : values.filter(isPresent).length);
                     break;
                 case "sum":
                 case "avg": {
                     const nums = values.filter((v) => typeof v === "number");
                     skipped += values.filter(isPresent).length - nums.length;
                     const total = nums.reduce((a, b) => a + b, 0);
-                    out[agg.as] = agg.fn === "sum" ? total : nums.length === 0 ? null : total / nums.length;
+                    setOwn(out, agg.as, agg.fn === "sum" ? total : nums.length === 0 ? null : total / nums.length);
                     break;
                 }
                 case "min":
                 case "max": {
                     const present = values.filter(isPresent);
                     if (present.length === 0) {
-                        out[agg.as] = null;
+                        setOwn(out, agg.as, null);
                         break;
                     }
-                    out[agg.as] = present.reduce((best, v) => {
+                    setOwn(out, agg.as, present.reduce((best, v) => {
                         const cmp = totalCompare(v, best);
                         return (agg.fn === "min" ? cmp < 0 : cmp > 0) ? v : best;
-                    });
+                    }));
                     break;
                 }
                 case "first":
-                    out[agg.as] = values[0] ?? null;
+                    setOwn(out, agg.as, values[0] ?? null);
                     break;
                 case "last":
-                    out[agg.as] = values[values.length - 1] ?? null;
+                    setOwn(out, agg.as, values[values.length - 1] ?? null);
                     break;
                 case "distinct":
-                    out[agg.as] = new Set(values.filter(isPresent).map((v) => canonicalStringify(v))).size;
+                    setOwn(out, agg.as, new Set(values.filter(isPresent).map((v) => canonicalStringify(v))).size);
                     break;
                 default:
                     break;
@@ -337,7 +417,12 @@ function isPresent(v) {
  * how a join quietly loses rows.
  *
  * Every right-hand match is emitted, so a one-to-many join multiplies rows,
- * as SQL does. `maxRows` bounds that.
+ * as SQL does. `maxRows` bounds that — the WORK as well as the output: a row
+ * past the cap is counted, never built, so a join on a low-cardinality key
+ * (every row `status: "open"`) costs the rows it returns plus one pass over
+ * each side, not |left| x |right| merges thrown away. `totalRows` is the size
+ * the uncapped join would have had, counted arithmetically, so a caller can
+ * tell ten rows cut from eleven apart from ten cut from millions.
  */
 export function joinRecords(left, right, options) {
     const index = new Map();
@@ -355,13 +440,14 @@ export function joinRecords(left, right, options) {
     const rows = [];
     const matchedRight = new Set();
     let unmatchedLeft = 0;
-    let truncated = false;
-    const push = (row) => {
-        if (rows.length >= options.maxRows) {
-            truncated = true;
-            return;
-        }
-        rows.push(row);
+    let totalRows = 0;
+    // The cap is checked BEFORE a row is built. Building it first and then
+    // discarding it (the 0.7.0 shape) made maxRows bound the answer but not
+    // the work: every matching pair was merged whatever the cap said.
+    const keep = (build) => {
+        totalRows += 1;
+        if (rows.length < options.maxRows)
+            rows.push(build());
     };
     for (const l of left) {
         const key = getPath(l, options.leftKey);
@@ -370,13 +456,16 @@ export function joinRecords(left, right, options) {
         if (matches === undefined || matches.length === 0) {
             unmatchedLeft += 1;
             if (options.kind === "left" || options.kind === "full")
-                push({ ...l });
+                keep(() => ({ ...l }));
             continue;
         }
         if (id !== null)
             matchedRight.add(id);
-        for (const r of matches)
-            push(mergeRow(l, r, options.rightPrefix));
+        const take = Math.min(Math.max(options.maxRows - rows.length, 0), matches.length);
+        for (let i = 0; i < take; i++) {
+            rows.push(mergeRow(l, matches[i], options.rightPrefix));
+        }
+        totalRows += matches.length;
     }
     let unmatchedRight = 0;
     for (const r of right) {
@@ -385,15 +474,21 @@ export function joinRecords(left, right, options) {
         if (id === null || !matchedRight.has(id)) {
             unmatchedRight += 1;
             if (options.kind === "right" || options.kind === "full")
-                push({ ...r });
+                keep(() => ({ ...r }));
         }
     }
-    return { rows, truncated, unmatchedLeft, unmatchedRight };
+    return {
+        rows,
+        truncated: totalRows > rows.length,
+        totalRows,
+        unmatchedLeft,
+        unmatchedRight,
+    };
 }
 function mergeRow(left, right, prefix) {
     const out = { ...left };
     for (const k of Object.keys(right)) {
-        out[Object.hasOwn(left, k) ? `${prefix}${k}` : k] = right[k];
+        setOwn(out, Object.hasOwn(left, k) ? `${prefix}${k}` : k, right[k]);
     }
     return out;
 }
@@ -413,7 +508,7 @@ export function recordsToColumns(records) {
     }
     const out = {};
     for (const k of keys)
-        out[k] = records.map((r) => (Object.hasOwn(r, k) ? r[k] : null));
+        setOwn(out, k, records.map((r) => (Object.hasOwn(r, k) ? r[k] : null)));
     return out;
 }
 /**
@@ -430,25 +525,35 @@ export function columnsToRecords(columns) {
         const rec = {};
         for (const k of keys) {
             const col = columns[k] ?? [];
-            rec[k] = i < col.length ? col[i] : null;
+            setOwn(rec, k, i < col.length ? col[i] : null);
         }
         records.push(rec);
     }
     return { records, ragged };
 }
 /** Flatten nested objects into dotted keys. Arrays become `a.0`, `a.1` when expanded. */
-export function flattenObject(value, separator, expandArrays, maxDepth) {
+export function flattenObject(value, separator, expandArrays, maxDepth, maxKeyChars = Number.POSITIVE_INFINITY) {
     const out = {};
+    // Every flattened key spells its whole path, so the keys alone are
+    // depth x leaves characters; `maxKeyChars` stops the walk (throwing
+    // OutputLimitError) before they are all built.
+    let keyChars = 0;
+    const put = (key, v) => {
+        keyChars += key.length;
+        if (keyChars > maxKeyChars)
+            throw new OutputLimitError(maxKeyChars, "the flattened keys");
+        setOwn(out, key, v);
+    };
     const walk = (node, prefix, depth) => {
         if (depth >= maxDepth) {
-            out[prefix] = node;
+            put(prefix, node);
             return;
         }
         if (isPlainObject(node)) {
             const keys = Object.keys(node);
             if (keys.length === 0) {
                 if (prefix !== "")
-                    out[prefix] = {};
+                    put(prefix, {});
                 return;
             }
             for (const k of keys)
@@ -458,13 +563,13 @@ export function flattenObject(value, separator, expandArrays, maxDepth) {
         if (Array.isArray(node) && expandArrays) {
             if (node.length === 0) {
                 if (prefix !== "")
-                    out[prefix] = [];
+                    put(prefix, []);
                 return;
             }
             node.forEach((el, i) => walk(el, prefix === "" ? String(i) : `${prefix}${separator}${i}`, depth + 1));
             return;
         }
-        out[prefix] = node;
+        put(prefix, node);
     };
     walk(value, "", 0);
     return out;
@@ -482,17 +587,17 @@ export function unflattenObject(flat, separator, arraysFromNumericKeys) {
         let cur = root;
         for (let i = 0; i < segs.length - 1; i++) {
             const seg = segs[i];
-            const next = cur[seg];
+            const next = getOwn(cur, seg);
             if (!isPlainObject(next)) {
                 const fresh = {};
-                cur[seg] = fresh;
+                setOwn(cur, seg, fresh);
                 cur = fresh;
             }
             else {
                 cur = next;
             }
         }
-        cur[segs[segs.length - 1]] = flat[key];
+        setOwn(cur, segs[segs.length - 1], flat[key]);
     }
     return arraysFromNumericKeys ? arrayify(root) : root;
 }
@@ -502,7 +607,7 @@ function arrayify(node) {
     const keys = Object.keys(node);
     const converted = {};
     for (const k of keys)
-        converted[k] = arrayify(node[k]);
+        setOwn(converted, k, arrayify(node[k]));
     if (keys.length === 0)
         return converted;
     const allNumeric = keys.every((k) => /^(0|[1-9][0-9]*)$/.test(k));

package/dist/lib/toml.d.ts

@@ -43,6 +43,13 @@ export declare class TomlError extends Error {
     readonly line: number;
     constructor(message: string, line: number);
 }
+/**
+ * A key or table path with more parts than MAX_NESTING_DEPTH, refused as it
+ * is read: the value it builds would nest past the cap and be refused
+ * anyway, and building it first cost the square of the path's length.
+ */
+export declare class TomlTooDeepError extends TomlError {
+}
 /** Parse a TOML document into JSON values. Throws `TomlError` with a line number. */
 export declare function parseToml(text: string): Record<string, unknown>;
 /** Type a bare TOML token. Exported so the tests can pin the exact rules. */
@@ -56,4 +63,4 @@ export type TomlWriteResult = {
  * keys that were dropped because TOML has no representation for them
  * (`null` and `undefined`).
  */
-export declare function stringifyToml(value: unknown): TomlWriteResult;
+export declare function stringifyToml(value: unknown, maxChars?: number): TomlWriteResult;