@crewhaus/tool-data 0.7.0 → 0.8.0

package/dist/lib/jsonpath.js

@@ -33,7 +33,8 @@
  * `&&`. Anything unsupported raises a parse error naming the offending
  * position, never a silent empty result.
  */
-import { isPlainObject } from "./json";
+import { testPatternSync } from "@crewhaus/tool-schema";
+import { getOwn, isPlainObject } from "./json";
 export class PathError extends Error {
 }
 const IDENT = /^[A-Za-z_$][A-Za-z0-9_$-]*/;
@@ -233,85 +234,160 @@ function parseLiteral(text, whole, at) {
         return n;
     throw new PathError(`cannot read ${JSON.stringify(text)} as a literal at position ${at} of ${JSON.stringify(whole)}`);
 }
-/** Run a parsed path over a value, returning matches in document order. */
-export function queryPath(root, steps, limit) {
-    let nodes = [{ path: [], value: root }];
-    for (const step of steps) {
-        const next = [];
-        for (const node of nodes)
-            applyStep(step, node, next);
-        nodes = next;
-        // Bound intermediate sets too: `..*` over a deep tree can blow up.
-        const ceiling = Math.max(limit, 1) * 20;
-        if (nodes.length > ceiling)
-            nodes = nodes.slice(0, ceiling);
+/**
+ * The most nodes one query may visit, counting every node a step produces
+ * and every node a `..` walk passes. A single pass over the largest document
+ * this package accepts is well inside it; a path that walks the document
+ * again for each node it found (`$..*..x` over a deep document) is what it
+ * stops.
+ */
+export const MAX_QUERY_VISITS = 5_000_000;
+/**
+ * Run a parsed path over a value, returning matches in document order.
+ *
+ * Evaluated lazily, depth first: each node a step produces runs through the
+ * remaining steps before the next is produced, which gives exactly the
+ * step-by-step, document-order result, and lets the query stop as soon as it
+ * has one match more than `limit` (to know it truncated). 0.7.0 built each
+ * step's whole node list first and then cut it at 20 x limit — so `$..*`
+ * materialized every node with its path, and a cut in the middle of a path
+ * silently dropped real matches: `$[*].b` over thirty records with `b` only
+ * in the last answered zero matches, not truncated.
+ */
+export function queryPath(root, steps, limit, maxVisits = MAX_QUERY_VISITS, regex) {
+    const found = [];
+    const want = Math.max(limit, 0) + 1;
+    const budget = { left: maxVisits };
+    const filters = { regex, undetermined: [] };
+    const start = { value: root, parent: null, seg: null };
+    let stopped = false;
+    if (steps.length === 0) {
+        found.push(start);
+    }
+    else {
+        // One iterator per step: iterators[i] yields step i's outputs for the
+        // node the level above handed it. An explicit stack, not recursion, so a
+        // long path cannot overflow it.
+        const iterators = [expand(steps[0], start, budget, filters)];
+        while (iterators.length > 0) {
+            const next = iterators[iterators.length - 1].next();
+            if (budget.left < 0) {
+                stopped = true;
+                break;
+            }
+            if (next.done === true) {
+                iterators.pop();
+                continue;
+            }
+            if (iterators.length === steps.length) {
+                found.push(next.value);
+                if (found.length >= want)
+                    break;
+                continue;
+            }
+            iterators.push(expand(steps[iterators.length], next.value, budget, filters));
+        }
     }
-    const truncated = nodes.length > limit;
+    const shown = found.slice(0, Math.max(limit, 0));
+    const open = filters.undetermined;
     return {
-        matches: nodes.slice(0, limit).map((n) => ({ path: renderPath(n.path), value: n.value })),
-        truncated,
+        matches: shown.map((n) => ({ path: renderPath(segmentsOf(n)), value: n.value })),
+        truncated: stopped || found.length > shown.length,
+        ...(stopped ? { stoppedAtVisits: maxVisits } : {}),
+        ...(open.length > 0
+            ? {
+                undetermined: {
+                    count: open.length,
+                    paths: open.slice(0, 50).map((n) => renderPath(segmentsOf(n))),
+                },
+            }
+            : {}),
     };
 }
-function applyStep(step, node, out) {
+function segmentsOf(node) {
+    const out = [];
+    for (let n = node; n !== null && n.seg !== null; n = n.parent)
+        out.push(n.seg);
+    return out.reverse();
+}
+const child = (parent, seg, value) => ({ value, parent, seg });
+/** The nodes one step produces from `node`, lazily, charging each to the budget. */
+function* expand(step, node, budget, filters) {
     const v = node.value;
     switch (step.kind) {
         case "child": {
             if (isPlainObject(v) && Object.hasOwn(v, step.name)) {
-                out.push({ path: [...node.path, step.name], value: v[step.name] });
+                budget.left -= 1;
+                yield child(node, step.name, v[step.name]);
             }
             return;
         }
         case "wildcard": {
             if (Array.isArray(v)) {
-                v.forEach((el, i) => out.push({ path: [...node.path, i], value: el }));
+                for (let i = 0; i < v.length; i++) {
+                    budget.left -= 1;
+                    yield child(node, i, v[i]);
+                }
             }
             else if (isPlainObject(v)) {
-                for (const k of Object.keys(v))
-                    out.push({ path: [...node.path, k], value: v[k] });
+                for (const k of Object.keys(v)) {
+                    budget.left -= 1;
+                    yield child(node, k, v[k]);
+                }
             }
             return;
         }
         case "descend": {
             const name = step.name;
-            walk(node, (n) => {
+            for (const n of descendants(node, budget)) {
                 if (name === null) {
-                    if (n.path.length !== node.path.length)
-                        out.push(n);
-                    return;
+                    if (n !== node)
+                        yield n;
                 }
-                if (isPlainObject(n.value) && Object.hasOwn(n.value, name)) {
-                    out.push({ path: [...n.path, name], value: n.value[name] });
+                else if (isPlainObject(n.value) && Object.hasOwn(n.value, name)) {
+                    yield child(n, name, n.value[name]);
                 }
-            });
+            }
             return;
         }
         case "index": {
             if (!Array.isArray(v))
                 return;
             const i = step.index < 0 ? v.length + step.index : step.index;
-            if (i >= 0 && i < v.length)
-                out.push({ path: [...node.path, i], value: v[i] });
+            if (i >= 0 && i < v.length) {
+                budget.left -= 1;
+                yield child(node, i, v[i]);
+            }
             return;
         }
         case "slice": {
             if (!Array.isArray(v))
                 return;
             for (const i of sliceIndices(v.length, step.start, step.end, step.step)) {
-                out.push({ path: [...node.path, i], value: v[i] });
+                budget.left -= 1;
+                yield child(node, i, v[i]);
             }
             return;
         }
         case "filter": {
+            const keep = (seg, value) => {
+                const verdict = matchesFilter(value, step, filters.regex);
+                if (verdict === "undetermined")
+                    filters.undetermined.push(child(node, seg, value));
+                return verdict === true;
+            };
             if (Array.isArray(v)) {
-                v.forEach((el, i) => {
-                    if (matchesFilter(el, step))
-                        out.push({ path: [...node.path, i], value: el });
-                });
+                for (let i = 0; i < v.length; i++) {
+                    budget.left -= 1;
+                    if (keep(i, v[i]))
+                        yield child(node, i, v[i]);
+                }
             }
             else if (isPlainObject(v)) {
                 for (const k of Object.keys(v)) {
-                    if (matchesFilter(v[k], step))
-                        out.push({ path: [...node.path, k], value: v[k] });
+                    budget.left -= 1;
+                    if (keep(k, v[k]))
+                        yield child(node, k, v[k]);
                 }
             }
             return;
@@ -320,16 +396,41 @@ function applyStep(step, node, out) {
             return;
     }
 }
-/** Depth-first pre-order walk, used by recursive descent. */
-function walk(node, visit) {
-    visit(node);
-    const v = node.value;
-    if (Array.isArray(v)) {
-        v.forEach((el, i) => walk({ path: [...node.path, i], value: el }, visit));
-    }
-    else if (isPlainObject(v)) {
-        for (const k of Object.keys(v))
-            walk({ path: [...node.path, k], value: v[k] }, visit);
+/**
+ * `node` and everything under it, depth-first pre-order, lazily. An explicit
+ * stack of child cursors, one per level, so a deep document costs one frame
+ * per level and never a recursion.
+ */
+function* descendants(node, budget) {
+    budget.left -= 1;
+    yield node;
+    const cursorOf = (n) => Array.isArray(n.value)
+        ? { of: n, keys: null, at: 0 }
+        : isPlainObject(n.value)
+            ? { of: n, keys: Object.keys(n.value), at: 0 }
+            : null;
+    const stack = [];
+    const first = cursorOf(node);
+    if (first !== null)
+        stack.push(first);
+    while (stack.length > 0) {
+        const top = stack[stack.length - 1];
+        const container = top.of.value;
+        const size = top.keys === null ? container.length : top.keys.length;
+        if (top.at >= size) {
+            stack.pop();
+            continue;
+        }
+        const at = top.at;
+        top.at += 1;
+        const next = top.keys === null
+            ? child(top.of, at, container[at])
+            : child(top.of, top.keys[at], container[top.keys[at]]);
+        budget.left -= 1;
+        yield next;
+        const deeper = cursorOf(next);
+        if (deeper !== null)
+            stack.push(deeper);
     }
 }
 /** Python slice semantics, including negative bounds and a negative step. */
@@ -350,11 +451,19 @@ export function sliceIndices(length, start, end, step) {
     }
     return out;
 }
-function matchesFilter(value, step) {
+/**
+ * Does `value` pass the filter? A `=~` pattern is never run here: its
+ * answer comes from `regex`, resolved in the worker by the tool, and one
+ * not answered yet reads as a miss for this pass (the tool runs the query
+ * again once it is answered). One that could not be answered is
+ * `"undetermined"`; one the screen refuses, or that does not compile, is a
+ * bad path, where 0.7.0 silently matched nothing.
+ */
+function matchesFilter(value, step, regex) {
     let cur = value;
     for (const seg of step.field) {
         if (isPlainObject(cur))
-            cur = cur[seg];
+            cur = getOwn(cur, seg);
         else if (Array.isArray(cur) && /^-?\d+$/.test(seg)) {
             const i = Number(seg);
             cur = cur[i < 0 ? cur.length + i : i];
@@ -372,12 +481,16 @@ function matchesFilter(value, step) {
     if (step.op === "=~") {
         if (typeof lit !== "string")
             return false;
-        try {
-            return new RegExp(lit).test(String(cur));
-        }
-        catch {
+        const subject = String(cur);
+        const answer = regex === undefined ? testPatternSync(lit, "", subject) : regex.lookup(lit, "", subject);
+        if (answer === undefined)
             return false;
+        if (typeof answer === "boolean")
+            return answer;
+        if ("refused" in answer) {
+            throw new PathError(`the filter's regex /${lit}/ was not run: ${answer.refused}`);
         }
+        return "undetermined";
     }
     if (typeof cur === "number" && typeof lit === "number") {
         return compare(cur < lit, cur > lit, step.op);

package/dist/lib/own-record.d.ts

@@ -0,0 +1,22 @@
+/**
+ * A `z.record` that keeps a key named `__proto__`.
+ *
+ * zod rebuilds a record into a fresh object and skips `__proto__` while it
+ * does (so the rebuild cannot set the new object's prototype). That is the
+ * right call for zod, and the wrong one for a data tool: `{"__proto__": 1}`
+ * is valid JSON, `JSON.parse` keeps it as an own property, and CsvParse,
+ * FlattenObject and JsonParse hand it back as one. Passed on to TableQuery,
+ * CsvWrite or UnflattenObject, the field vanished before the tool ran, with
+ * no error: the documented FlattenObject -> UnflattenObject round trip lost
+ * it, and a `__proto__` column read by CsvParse was not written by CsvWrite.
+ *
+ * `ownRecord(values)` is a `ZodRecord` in every way a reader of the schema
+ * can see (the model is shown the same JSON Schema, and introspection finds
+ * a record), so only the parse differs: after zod's own checks, an own
+ * `__proto__` key is validated with the same value schema and put back as
+ * an own data property, at its place in the key order. Nothing is ever
+ * assigned through `obj.__proto__`, so no prototype changes.
+ */
+import { z } from "zod";
+/** `z.record(values)`, except that a key named `__proto__` is kept as data. */
+export declare function ownRecord<Value extends z.ZodTypeAny>(values: Value): z.ZodRecord<z.ZodString, Value>;

package/dist/lib/own-record.js

@@ -0,0 +1,66 @@
+/**
+ * A `z.record` that keeps a key named `__proto__`.
+ *
+ * zod rebuilds a record into a fresh object and skips `__proto__` while it
+ * does (so the rebuild cannot set the new object's prototype). That is the
+ * right call for zod, and the wrong one for a data tool: `{"__proto__": 1}`
+ * is valid JSON, `JSON.parse` keeps it as an own property, and CsvParse,
+ * FlattenObject and JsonParse hand it back as one. Passed on to TableQuery,
+ * CsvWrite or UnflattenObject, the field vanished before the tool ran, with
+ * no error: the documented FlattenObject -> UnflattenObject round trip lost
+ * it, and a `__proto__` column read by CsvParse was not written by CsvWrite.
+ *
+ * `ownRecord(values)` is a `ZodRecord` in every way a reader of the schema
+ * can see (the model is shown the same JSON Schema, and introspection finds
+ * a record), so only the parse differs: after zod's own checks, an own
+ * `__proto__` key is validated with the same value schema and put back as
+ * an own data property, at its place in the key order. Nothing is ever
+ * assigned through `obj.__proto__`, so no prototype changes.
+ */
+import { z } from "zod";
+const PROTO_KEY = "__proto__";
+function isRecordLike(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+class ZodOwnRecord extends z.ZodRecord {
+    _parse(input) {
+        const result = super._parse(input);
+        const raw = input.data;
+        if (!isRecordLike(raw) || !Object.hasOwn(raw, PROTO_KEY))
+            return result;
+        const restore = (parsed) => {
+            if (parsed.status === "aborted")
+                return parsed;
+            const ownValue = this._def.valueType.safeParse(raw[PROTO_KEY]);
+            if (!ownValue.success) {
+                const ctx = this._getOrReturnCtx(input);
+                for (const issue of ownValue.error.issues) {
+                    z.addIssueToContext(ctx, { ...issue, path: [...input.path, PROTO_KEY, ...issue.path] });
+                }
+                return z.INVALID;
+            }
+            // Rebuild in the caller's key order, with `__proto__` as a plain own
+            // property where it stood.
+            const out = {};
+            for (const key of Object.keys(raw)) {
+                const value = key === PROTO_KEY ? ownValue.data : parsed.value[key];
+                Object.defineProperty(out, key, {
+                    value,
+                    enumerable: true,
+                    writable: true,
+                    configurable: true,
+                });
+            }
+            return { status: parsed.status, value: out };
+        };
+        return result instanceof Promise ? result.then(restore) : restore(result);
+    }
+}
+/** `z.record(values)`, except that a key named `__proto__` is kept as data. */
+export function ownRecord(values) {
+    return new ZodOwnRecord({
+        keyType: z.string(),
+        valueType: values,
+        typeName: z.ZodFirstPartyTypeKind.ZodRecord,
+    });
+}

package/dist/lib/patch.d.ts

@@ -51,16 +51,32 @@ export type PatchOp = {
     path: string;
     value: unknown;
 };
+/**
+ * The most values a patch may add to a document, over all its operations:
+ * about as many as the largest document this package reads holds. Each
+ * `copy` of the root doubles the document, so twenty of them on a
+ * seven-character document made 13.6 million characters.
+ */
+export declare const MAX_PATCH_ADDED_NODES = 2000000;
 /**
  * Apply an RFC 6902 patch. Returns the new document; the input is never
  * mutated. Throws `PatchError` (carrying the failing operation's index) if
- * any operation cannot be applied, leaving nothing half-done.
+ * any operation cannot be applied, leaving nothing half-done — including an
+ * operation that would take the values the patch adds (by add, replace or
+ * copy) past `maxAddedNodes`.
  */
-export declare function applyJsonPatch(doc: unknown, ops: ReadonlyArray<PatchOp>): unknown;
+export declare function applyJsonPatch(doc: unknown, ops: ReadonlyArray<PatchOp>, maxAddedNodes?: number): unknown;
 /**
  * Apply an RFC 7386 merge patch: an object merges key by key, `null` deletes
  * a key, and any non-object patch replaces the target outright. Arrays are
  * replaced whole — that is the RFC's behaviour, not a shortcut here.
+ *
+ * The result is a fresh value that shares nothing with either input, and
+ * every node of it is built exactly once: a target key the patch leaves
+ * alone is cloned, and one it touches is merged from the ORIGINAL child.
+ * (0.7.0 deep-cloned the target and then recursed into the clone, cloning
+ * the rest of the tree again at every level — quadratic in depth.) Keys keep
+ * the target's order, with new keys after them in the patch's order.
  */
 export declare function applyMergePatch(target: unknown, patch: unknown): unknown;
 /**

package/dist/lib/patch.js

@@ -7,7 +7,7 @@
  * leaves the caller's input untouched and reports which operation index
  * failed — which is the behaviour RFC 6902 §5 requires.
  */
-import { deepClone, deepEqual, isPlainObject } from "./json";
+import { deepClone, deepEqual, isPlainObject, setOwn } from "./json";
 export class PatchError extends Error {
     opIndex;
     constructor(message, opIndex) {
@@ -109,7 +109,9 @@ function addAt(parent, token, value, pointer, i) {
         return;
     }
     if (isPlainObject(parent)) {
-        parent[token] = value;
+        // An own member even for `__proto__`: assignment would call the setter,
+        // change nothing, and report success.
+        setOwn(parent, token, value);
         return;
     }
     throw new PatchError(`cannot add to a ${typeof parent} at "${pointer}"`, i);
@@ -135,13 +137,45 @@ function removeAt(parent, token, pointer, i) {
     }
     throw new PatchError(`cannot remove from a ${typeof parent} at "${pointer}"`, i);
 }
+/**
+ * The most values a patch may add to a document, over all its operations:
+ * about as many as the largest document this package reads holds. Each
+ * `copy` of the root doubles the document, so twenty of them on a
+ * seven-character document made 13.6 million characters.
+ */
+export const MAX_PATCH_ADDED_NODES = 2_000_000;
+/** Charge the values in `value` to the patch's budget; throw once it is spent. */
+function chargeNodes(value, budget, opIndex) {
+    const stack = [value];
+    while (stack.length > 0) {
+        const node = stack.pop();
+        budget.left -= 1;
+        if (budget.left < 0) {
+            throw new PatchError(`the patch would add more than ${MAX_PATCH_ADDED_NODES} values to the document`, opIndex);
+        }
+        if (Array.isArray(node))
+            for (const el of node)
+                stack.push(el);
+        else if (isPlainObject(node))
+            for (const k of Object.keys(node))
+                stack.push(node[k]);
+    }
+}
 /**
  * Apply an RFC 6902 patch. Returns the new document; the input is never
  * mutated. Throws `PatchError` (carrying the failing operation's index) if
- * any operation cannot be applied, leaving nothing half-done.
+ * any operation cannot be applied, leaving nothing half-done — including an
+ * operation that would take the values the patch adds (by add, replace or
+ * copy) past `maxAddedNodes`.
  */
-export function applyJsonPatch(doc, ops) {
+export function applyJsonPatch(doc, ops, maxAddedNodes = MAX_PATCH_ADDED_NODES) {
     let working = deepClone(doc);
+    const budget = { left: maxAddedNodes };
+    // Every value an operation adds is counted before it is cloned in.
+    const added = (value, i) => {
+        chargeNodes(value, budget, i);
+        return deepClone(value);
+    };
     ops.forEach((op, i) => {
         switch (op.op) {
             case "test": {
@@ -155,11 +189,11 @@ export function applyJsonPatch(doc, ops) {
             }
             case "add": {
                 if (op.path === "") {
-                    working = deepClone(op.value);
+                    working = added(op.value, i);
                     return;
                 }
                 const { parent, token } = locateParent(working, op.path, i);
-                addAt(parent, token, deepClone(op.value), op.path, i);
+                addAt(parent, token, added(op.value, i), op.path, i);
                 return;
             }
             case "remove": {
@@ -171,7 +205,7 @@ export function applyJsonPatch(doc, ops) {
             }
             case "replace": {
                 if (op.path === "") {
-                    working = deepClone(op.value);
+                    working = added(op.value, i);
                     return;
                 }
                 const existing = resolvePointer(working, op.path);
@@ -180,7 +214,7 @@ export function applyJsonPatch(doc, ops) {
                 }
                 const { parent, token } = locateParent(working, op.path, i);
                 removeAt(parent, token, op.path, i);
-                addAt(parent, token, deepClone(op.value), op.path, i);
+                addAt(parent, token, added(op.value, i), op.path, i);
                 return;
             }
             case "move": {
@@ -210,11 +244,11 @@ export function applyJsonPatch(doc, ops) {
                 if (!src.found)
                     throw new PatchError(`cannot copy from "${op.from}": ${src.reason}`, i);
                 if (op.path === "") {
-                    working = deepClone(src.value);
+                    working = added(src.value, i);
                     return;
                 }
                 const dst = locateParent(working, op.path, i);
-                addAt(dst.parent, dst.token, deepClone(src.value), op.path, i);
+                addAt(dst.parent, dst.token, added(src.value, i), op.path, i);
                 return;
             }
             default: {
@@ -229,21 +263,38 @@ export function applyJsonPatch(doc, ops) {
  * Apply an RFC 7386 merge patch: an object merges key by key, `null` deletes
  * a key, and any non-object patch replaces the target outright. Arrays are
  * replaced whole — that is the RFC's behaviour, not a shortcut here.
+ *
+ * The result is a fresh value that shares nothing with either input, and
+ * every node of it is built exactly once: a target key the patch leaves
+ * alone is cloned, and one it touches is merged from the ORIGINAL child.
+ * (0.7.0 deep-cloned the target and then recursed into the clone, cloning
+ * the rest of the tree again at every level — quadratic in depth.) Keys keep
+ * the target's order, with new keys after them in the patch's order.
  */
 export function applyMergePatch(target, patch) {
     if (!isPlainObject(patch))
         return deepClone(patch);
-    const base = isPlainObject(target)
-        ? deepClone(target)
-        : {};
+    const out = {};
+    const source = isPlainObject(target) ? target : null;
+    if (source !== null) {
+        for (const key of Object.keys(source)) {
+            if (!Object.hasOwn(patch, key)) {
+                setOwn(out, key, deepClone(source[key]));
+                continue;
+            }
+            const value = patch[key];
+            if (value !== null)
+                setOwn(out, key, applyMergePatch(source[key], value));
+        }
+    }
     for (const key of Object.keys(patch)) {
+        if (source !== null && Object.hasOwn(source, key))
+            continue;
         const value = patch[key];
-        if (value === null)
-            delete base[key];
-        else
-            base[key] = applyMergePatch(base[key], value);
+        if (value !== null)
+            setOwn(out, key, applyMergePatch(undefined, value));
     }
-    return base;
+    return out;
 }
 /**
  * Derive the RFC 7386 merge patch that turns `from` into `to`. Returns the
@@ -257,16 +308,16 @@ export function diffMergePatch(from, to) {
     const out = {};
     for (const key of Object.keys(to)) {
         if (!Object.hasOwn(from, key)) {
-            out[key] = deepClone(to[key]);
+            setOwn(out, key, deepClone(to[key]));
             continue;
         }
         if (deepEqual(from[key], to[key]))
             continue;
-        out[key] = diffMergePatch(from[key], to[key]);
+        setOwn(out, key, diffMergePatch(from[key], to[key]));
     }
     for (const key of Object.keys(from)) {
         if (!Object.hasOwn(to, key))
-            out[key] = null;
+            setOwn(out, key, null);
     }
     return out;
 }