@crewhaus/tool-data 0.7.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/lib/csv.d.ts CHANGED
@@ -31,6 +31,20 @@ export declare class CsvError extends Error {
31
31
  readonly line: number;
32
32
  constructor(message: string, line: number);
33
33
  }
34
+ /**
35
+ * Why a delimiter/quote pair cannot describe a CSV file, or null when it can.
36
+ *
37
+ * One check for the reader and the writer, so neither accepts a dialect the
38
+ * other cannot honour. A line break cannot be the delimiter or the quote:
39
+ * rows are separated by line breaks, so a `\n` delimiter turns every field
40
+ * into a row (a writer's output read back as one header line of values and
41
+ * zero records), and a `\r` one splits CRLF rows into phantom columns. The
42
+ * quote character cannot be the delimiter, or a quoted field cannot be told
43
+ * from a field boundary. U+FEFF cannot be either: a reader drops it from the
44
+ * start of a file as a byte-order mark, so `["", "x"]` written with it as
45
+ * the delimiter read back as `["x"]`.
46
+ */
47
+ export declare function csvDialectError(delimiter: string, quote: string): string | null;
34
48
  /** Split CSV text into rows of raw string fields. Throws `CsvError` on an unterminated quote. */
35
49
  export declare function parseCsv(text: string, options: CsvParseOptions): CsvParseResult;
36
50
  /**
@@ -65,6 +79,13 @@ export type CsvWriteOptions = {
65
79
  /** Quote every field, not only the ones that need it. */
66
80
  quoteAll: boolean;
67
81
  header: string[] | null;
82
+ /**
83
+ * Throw `OutputLimitError` once the text would pass this many characters.
84
+ * Records with keys that differ make a column for every key and a cell for
85
+ * every column in every row, so the output can be rows x columns even
86
+ * when the input was rows + columns.
87
+ */
88
+ maxChars?: number;
68
89
  };
69
90
  /** Quote a single field if it contains the delimiter, a quote, or a line break. */
70
91
  export declare function escapeCsvField(value: string, options: CsvWriteOptions): string;
@@ -74,7 +95,11 @@ export declare function escapeCsvField(value: string, options: CsvWriteOptions):
74
95
  * `[object Object]` is how a pipeline loses data without noticing.
75
96
  */
76
97
  export declare function cellToString(value: unknown): string;
77
- /** Serialize rows of already-stringified cells. */
78
- export declare function writeCsvRows(rows: ReadonlyArray<ReadonlyArray<unknown>>, options: CsvWriteOptions): string;
98
+ /**
99
+ * Serialize rows of already-stringified cells. Throws `CsvError` for a
100
+ * dialect `parseCsv` would refuse or misread (see `csvDialectError`), so
101
+ * nothing is written that this package cannot read back.
102
+ */
103
+ export declare function writeCsvRows(rows: Iterable<ReadonlyArray<unknown>>, options: CsvWriteOptions): string;
79
104
  /** Column order for a set of records: keys in first-seen order across every record. */
80
105
  export declare function unionKeys(records: ReadonlyArray<Record<string, unknown>>): string[];
package/dist/lib/csv.js CHANGED
@@ -13,6 +13,7 @@
13
13
  * caller's problem, and there is no support for backslash escapes, which
14
14
  * RFC 4180 does not define.
15
15
  */
16
+ import { OutputLimitError, setOwn } from "./json";
16
17
  export class CsvError extends Error {
17
18
  line;
18
19
  constructor(message, line) {
@@ -20,15 +21,43 @@ export class CsvError extends Error {
20
21
  this.line = line;
21
22
  }
22
23
  }
23
- /** Split CSV text into rows of raw string fields. Throws `CsvError` on an unterminated quote. */
24
- export function parseCsv(text, options) {
25
- const { delimiter, quote } = options;
24
+ /**
25
+ * Why a delimiter/quote pair cannot describe a CSV file, or null when it can.
26
+ *
27
+ * One check for the reader and the writer, so neither accepts a dialect the
28
+ * other cannot honour. A line break cannot be the delimiter or the quote:
29
+ * rows are separated by line breaks, so a `\n` delimiter turns every field
30
+ * into a row (a writer's output read back as one header line of values and
31
+ * zero records), and a `\r` one splits CRLF rows into phantom columns. The
32
+ * quote character cannot be the delimiter, or a quoted field cannot be told
33
+ * from a field boundary. U+FEFF cannot be either: a reader drops it from the
34
+ * start of a file as a byte-order mark, so `["", "x"]` written with it as
35
+ * the delimiter read back as `["x"]`.
36
+ */
37
+ export function csvDialectError(delimiter, quote) {
26
38
  if (delimiter.length !== 1)
27
- throw new CsvError("delimiter must be a single character", 0);
39
+ return "delimiter must be a single character";
28
40
  if (quote.length !== 1)
29
- throw new CsvError("quote must be a single character", 0);
41
+ return "quote must be a single character";
42
+ if (delimiter === "\ufeff" || quote === "\ufeff") {
43
+ return "neither delimiter nor quote can be U+FEFF — a reader drops it from the start of a file as a byte-order mark";
44
+ }
45
+ if (delimiter === "\r" || delimiter === "\n") {
46
+ return "delimiter cannot be a line break (CR or LF) — rows are separated by line breaks";
47
+ }
48
+ if (quote === "\r" || quote === "\n") {
49
+ return "quote cannot be a line break (CR or LF) — rows are separated by line breaks";
50
+ }
30
51
  if (delimiter === quote)
31
- throw new CsvError("delimiter and quote must differ", 0);
52
+ return "delimiter and quote must differ";
53
+ return null;
54
+ }
55
+ /** Split CSV text into rows of raw string fields. Throws `CsvError` on an unterminated quote. */
56
+ export function parseCsv(text, options) {
57
+ const { delimiter, quote } = options;
58
+ const dialect = csvDialectError(delimiter, quote);
59
+ if (dialect !== null)
60
+ throw new CsvError(dialect, 0);
32
61
  const src = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
33
62
  const rows = [];
34
63
  let row = [];
@@ -154,10 +183,10 @@ export function rowsToRecords(rows, header, infer, nullTokens) {
154
183
  header.forEach((name, j) => {
155
184
  const cell = row[j];
156
185
  if (cell === undefined) {
157
- rec[name] = null;
186
+ setOwn(rec, name, null);
158
187
  return;
159
188
  }
160
- rec[name] = infer ? inferScalar(cell, nullTokens) : cell;
189
+ setOwn(rec, name, infer ? inferScalar(cell, nullTokens) : cell);
161
190
  });
162
191
  records.push(rec);
163
192
  });
@@ -211,16 +240,50 @@ export function cellToString(value) {
211
240
  return value;
212
241
  if (typeof value === "number" || typeof value === "boolean")
213
242
  return String(value);
214
- return JSON.stringify(value);
243
+ // JSON.stringify answers undefined for a function or a symbol, which no
244
+ // JSON value is; an empty field beats a crash in the quoting step.
245
+ return JSON.stringify(value) ?? "";
215
246
  }
216
- /** Serialize rows of already-stringified cells. */
247
+ /**
248
+ * Serialize rows of already-stringified cells. Throws `CsvError` for a
249
+ * dialect `parseCsv` would refuse or misread (see `csvDialectError`), so
250
+ * nothing is written that this package cannot read back.
251
+ */
217
252
  export function writeCsvRows(rows, options) {
253
+ const dialect = csvDialectError(options.delimiter, options.quote);
254
+ if (dialect !== null)
255
+ throw new CsvError(dialect, 0);
256
+ const max = options.maxChars ?? Number.POSITIVE_INFINITY;
218
257
  const lines = [];
219
- if (options.header !== null) {
220
- lines.push(options.header.map((h) => escapeCsvField(h, options)).join(options.delimiter));
221
- }
258
+ let chars = -options.newline.length;
259
+ const push = (line) => {
260
+ chars += line.length + options.newline.length;
261
+ if (chars > max)
262
+ throw new OutputLimitError(max, "the CSV");
263
+ lines.push(line);
264
+ };
265
+ const q = options.quote;
266
+ // Two cases a reader would otherwise misread, so both are quoted: a row
267
+ // whose only field is empty is a blank line, which a reader skips, and a
268
+ // leading U+FEFF in the file's first field is taken for a byte-order mark
269
+ // and dropped.
270
+ const encode = (cells) => {
271
+ if (cells.length === 1 && cells[0] === "")
272
+ return q + q;
273
+ return cells
274
+ .map((cell, i) => i === 0 && lines.length === 0 && cell.startsWith("\ufeff")
275
+ ? q + cell.split(q).join(q + q) + q
276
+ : escapeCsvField(cell, options))
277
+ .join(options.delimiter);
278
+ };
279
+ if (options.header !== null)
280
+ push(encode(options.header));
222
281
  for (const row of rows) {
223
- lines.push(row.map((cell) => escapeCsvField(cellToString(cell), options)).join(options.delimiter));
282
+ // A row's cells are delimiter-separated at the least; checked before the
283
+ // row is built, so one enormous row is not built to be refused.
284
+ if (chars + row.length > max)
285
+ throw new OutputLimitError(max, "the CSV");
286
+ push(encode(row.map(cellToString)));
224
287
  }
225
288
  return lines.join(options.newline);
226
289
  }
package/dist/lib/diff.js CHANGED
@@ -7,7 +7,7 @@
7
7
  * wrong one for a list of records — for records, key the arrays first with
8
8
  * `keyArraysBy`, which matches elements by a field instead of by position.
9
9
  */
10
- import { deepEqual, isPlainObject, typeOf } from "./json";
10
+ import { deepEqual, getOwn, isPlainObject, typeOf } from "./json";
11
11
  import { formatPointer } from "./patch";
12
12
  /** Compare two values, returning a flat list of differences by JSON Pointer path. */
13
13
  export function deepDiff(a, b, options) {
@@ -119,7 +119,7 @@ function canKey(arr, key) {
119
119
  for (const el of arr) {
120
120
  if (!isPlainObject(el))
121
121
  return false;
122
- const v = el[key];
122
+ const v = getOwn(el, key);
123
123
  if (v === undefined || v === null || typeof v === "object")
124
124
  return false;
125
125
  const s = String(v);
@@ -133,7 +133,7 @@ function diffKeyedArrays(left, right, key, path, depth, recurse, push) {
133
133
  const index = (arr) => {
134
134
  const m = new Map();
135
135
  arr.forEach((el, i) => {
136
- const v = el[key];
136
+ const v = getOwn(el, key);
137
137
  m.set(String(v), { i, value: el });
138
138
  });
139
139
  return m;
@@ -11,6 +11,24 @@ export type JsonValue = null | boolean | number | string | JsonValue[] | {
11
11
  };
12
12
  /** A plain object, as distinct from an array or a primitive. */
13
13
  export declare function isPlainObject(value: unknown): value is Record<string, unknown>;
14
+ /**
15
+ * Read `key` only when it is the object's OWN property. A key taken from the
16
+ * data (a dotted path segment, a TOML table name, an XML element name) must
17
+ * never reach an inherited member: `constructor` would read
18
+ * `Object.prototype.constructor`, and `__proto__` would hand back
19
+ * `Object.prototype` itself as something to write into — which is how one
20
+ * hostile document polluted every object in the process.
21
+ */
22
+ export declare function getOwn(obj: Record<string, unknown>, key: string): unknown;
23
+ /**
24
+ * Write `key` as an own data property. For every key but `__proto__` plain
25
+ * assignment already does that; `__proto__` alone is an accessor inherited
26
+ * from `Object.prototype`, whose setter would replace the object's prototype
27
+ * (dropping the key) instead of storing it. `defineProperty` stores it as the
28
+ * data it is, so `{"__proto__": …}` round-trips like any other key — the
29
+ * way `JSON.parse` itself reads it.
30
+ */
31
+ export declare function setOwn(obj: Record<string, unknown>, key: string, value: unknown): void;
14
32
  /** Structural equality over JSON values. Key order is irrelevant; NaN never appears in JSON. */
15
33
  export declare function deepEqual(a: unknown, b: unknown): boolean;
16
34
  /** A structured clone that only has to handle JSON shapes, so no cycles exist. */
@@ -63,3 +81,36 @@ export declare function parseJson(text: string): {
63
81
  };
64
82
  /** A short, human-readable type name used in diffs and error messages. */
65
83
  export declare function typeOf(value: unknown): string;
84
+ /**
85
+ * The deepest nesting a document may have: each `[` or `{` is one level, a
86
+ * scalar document is depth 0. XmlParse's own ceiling, and far past any real
87
+ * configuration, API response or AST dump.
88
+ */
89
+ export declare const MAX_NESTING_DEPTH = 256;
90
+ /**
91
+ * Whether JSON text nests deeper than `limit`, from one linear pass over the
92
+ * characters: brackets inside string literals (escapes honoured) do not
93
+ * count, and nothing is parsed, so a hostile document is refused before a
94
+ * deep tree exists. Malformed text may be over-counted; it would fail to
95
+ * parse anyway.
96
+ */
97
+ export declare function jsonTextNestsDeeper(text: string, limit: number): boolean;
98
+ /** Whether a parsed value nests deeper than `limit`. Iterative: no recursion to overflow. */
99
+ export declare function nestingDepthExceeds(value: unknown, limit: number): boolean;
100
+ /** A result, or a part of one, that would be larger than its tool may return. */
101
+ export declare class OutputLimitError extends Error {
102
+ readonly limit: number;
103
+ constructor(limit: number, what?: string);
104
+ }
105
+ /**
106
+ * The length of `JSON.stringify(value, null, indent)`, computed without
107
+ * building it, and abandoned as soon as it passes `cap` (the answer is then
108
+ * some number above `cap`). Iterative and exact for JSON values; a
109
+ * non-plain object (none of this package's readers make one) is measured by
110
+ * serializing it.
111
+ *
112
+ * Measured first so that a result too large to return is refused before it
113
+ * exists: pretty-printing multiplies every line by depth x indent, and a
114
+ * record set can repeat a long key once per row.
115
+ */
116
+ export declare function jsonTextLength(value: unknown, indent: number, cap: number): number;
package/dist/lib/json.js CHANGED
@@ -10,6 +10,38 @@
10
10
  export function isPlainObject(value) {
11
11
  return typeof value === "object" && value !== null && !Array.isArray(value);
12
12
  }
13
+ /**
14
+ * Read `key` only when it is the object's OWN property. A key taken from the
15
+ * data (a dotted path segment, a TOML table name, an XML element name) must
16
+ * never reach an inherited member: `constructor` would read
17
+ * `Object.prototype.constructor`, and `__proto__` would hand back
18
+ * `Object.prototype` itself as something to write into — which is how one
19
+ * hostile document polluted every object in the process.
20
+ */
21
+ export function getOwn(obj, key) {
22
+ return Object.hasOwn(obj, key) ? obj[key] : undefined;
23
+ }
24
+ /**
25
+ * Write `key` as an own data property. For every key but `__proto__` plain
26
+ * assignment already does that; `__proto__` alone is an accessor inherited
27
+ * from `Object.prototype`, whose setter would replace the object's prototype
28
+ * (dropping the key) instead of storing it. `defineProperty` stores it as the
29
+ * data it is, so `{"__proto__": …}` round-trips like any other key — the
30
+ * way `JSON.parse` itself reads it.
31
+ */
32
+ export function setOwn(obj, key, value) {
33
+ if (key === "__proto__") {
34
+ Object.defineProperty(obj, key, {
35
+ value,
36
+ writable: true,
37
+ enumerable: true,
38
+ configurable: true,
39
+ });
40
+ }
41
+ else {
42
+ obj[key] = value;
43
+ }
44
+ }
13
45
  /** Structural equality over JSON values. Key order is irrelevant; NaN never appears in JSON. */
14
46
  export function deepEqual(a, b) {
15
47
  if (a === b)
@@ -45,7 +77,7 @@ export function deepClone(value) {
45
77
  if (isPlainObject(value)) {
46
78
  const out = {};
47
79
  for (const k of Object.keys(value))
48
- out[k] = deepClone(value[k]);
80
+ setOwn(out, k, deepClone(value[k]));
49
81
  return out;
50
82
  }
51
83
  return value;
@@ -74,7 +106,7 @@ export function sortKeysDeep(value) {
74
106
  if (isPlainObject(value)) {
75
107
  const out = {};
76
108
  for (const k of Object.keys(value).sort())
77
- out[k] = sortKeysDeep(value[k]);
109
+ setOwn(out, k, sortKeysDeep(value[k]));
78
110
  return out;
79
111
  }
80
112
  return value;
@@ -131,3 +163,143 @@ export function typeOf(value) {
131
163
  return "array";
132
164
  return typeof value;
133
165
  }
166
+ // ---------------------------------------------------------------------------
167
+ // Bounds on nesting and on output.
168
+ //
169
+ // A document's cost here is not its length. Every tool that walks a value
170
+ // does work per level as well as per node — a merge that cloned the rest of
171
+ // the tree at each level, a query that copied the path at each level, a
172
+ // pretty-printer that writes depth x indent spaces on every line — so a
173
+ // 48 KB document nested 8,000 deep cost gigabytes. Depth is capped at the
174
+ // door, and every result is measured before it is built.
175
+ /**
176
+ * The deepest nesting a document may have: each `[` or `{` is one level, a
177
+ * scalar document is depth 0. XmlParse's own ceiling, and far past any real
178
+ * configuration, API response or AST dump.
179
+ */
180
+ export const MAX_NESTING_DEPTH = 256;
181
+ /**
182
+ * Whether JSON text nests deeper than `limit`, from one linear pass over the
183
+ * characters: brackets inside string literals (escapes honoured) do not
184
+ * count, and nothing is parsed, so a hostile document is refused before a
185
+ * deep tree exists. Malformed text may be over-counted; it would fail to
186
+ * parse anyway.
187
+ */
188
+ export function jsonTextNestsDeeper(text, limit) {
189
+ let depth = 0;
190
+ let inString = false;
191
+ for (let i = 0; i < text.length; i++) {
192
+ const c = text.charCodeAt(i);
193
+ if (inString) {
194
+ // A backslash escapes the character after it.
195
+ if (c === 0x5c)
196
+ i += 1;
197
+ else if (c === 0x22)
198
+ inString = false;
199
+ continue;
200
+ }
201
+ if (c === 0x22)
202
+ inString = true;
203
+ else if (c === 0x5b || c === 0x7b) {
204
+ depth += 1;
205
+ if (depth > limit)
206
+ return true;
207
+ }
208
+ else if (c === 0x5d || c === 0x7d) {
209
+ if (depth > 0)
210
+ depth -= 1;
211
+ }
212
+ }
213
+ return false;
214
+ }
215
+ /** Whether a parsed value nests deeper than `limit`. Iterative: no recursion to overflow. */
216
+ export function nestingDepthExceeds(value, limit) {
217
+ const stack = [[value, 0]];
218
+ while (stack.length > 0) {
219
+ const [node, depth] = stack.pop();
220
+ const children = Array.isArray(node) ? node : isPlainObject(node) ? Object.values(node) : null;
221
+ if (children === null)
222
+ continue;
223
+ if (depth + 1 > limit)
224
+ return true;
225
+ for (const child of children) {
226
+ if (typeof child === "object" && child !== null)
227
+ stack.push([child, depth + 1]);
228
+ }
229
+ }
230
+ return false;
231
+ }
232
+ /** A result, or a part of one, that would be larger than its tool may return. */
233
+ export class OutputLimitError extends Error {
234
+ limit;
235
+ constructor(limit, what = "the result") {
236
+ super(`${what} would be more than ${limit} characters`);
237
+ this.limit = limit;
238
+ }
239
+ }
240
+ /** Nothing a value's JSON form renders as: skipped in an object, `null` in an array. */
241
+ const renders = (v) => v !== undefined && typeof v !== "function" && typeof v !== "symbol";
242
+ /**
243
+ * The length of `JSON.stringify(value, null, indent)`, computed without
244
+ * building it, and abandoned as soon as it passes `cap` (the answer is then
245
+ * some number above `cap`). Iterative and exact for JSON values; a
246
+ * non-plain object (none of this package's readers make one) is measured by
247
+ * serializing it.
248
+ *
249
+ * Measured first so that a result too large to return is refused before it
250
+ * exists: pretty-printing multiplies every line by depth x indent, and a
251
+ * record set can repeat a long key once per row.
252
+ */
253
+ export function jsonTextLength(value, indent, cap) {
254
+ if (!renders(value))
255
+ return 0;
256
+ const k = Math.min(10, Math.max(0, Math.floor(indent)));
257
+ let total = 0;
258
+ const stack = [[value, 0]];
259
+ while (stack.length > 0) {
260
+ const [node, depth] = stack.pop();
261
+ if (node === null || !renders(node)) {
262
+ total += 4; // null, or a non-rendering array element, which becomes null
263
+ }
264
+ else if (typeof node === "string") {
265
+ total += JSON.stringify(node).length;
266
+ }
267
+ else if (typeof node === "number") {
268
+ total += Number.isFinite(node) ? String(node).length : 4;
269
+ }
270
+ else if (typeof node === "boolean") {
271
+ total += node ? 4 : 5;
272
+ }
273
+ else if (Array.isArray(node)) {
274
+ const n = node.length;
275
+ if (n === 0)
276
+ total += 2;
277
+ else {
278
+ total += 2 + (n - 1) + (k === 0 ? 0 : n * (1 + (depth + 1) * k) + 1 + depth * k);
279
+ for (let i = n - 1; i >= 0; i--)
280
+ stack.push([node[i], depth + 1]);
281
+ }
282
+ }
283
+ else if (isPlainObject(node)) {
284
+ let m = 0;
285
+ for (const key of Object.keys(node)) {
286
+ const v = node[key];
287
+ if (!renders(v))
288
+ continue;
289
+ m += 1;
290
+ total += JSON.stringify(key).length + (k === 0 ? 1 : 2);
291
+ stack.push([v, depth + 1]);
292
+ }
293
+ if (m === 0)
294
+ total += 2;
295
+ else
296
+ total += 2 + (m - 1) + (k === 0 ? 0 : m * (1 + (depth + 1) * k) + 1 + depth * k);
297
+ }
298
+ else {
299
+ total += (JSON.stringify(node) ?? "null").length;
300
+ }
301
+ if (total > cap)
302
+ return total;
303
+ }
304
+ return total;
305
+ }
@@ -1,12 +1,3 @@
1
- /**
2
- * Line-delimited JSON. One value per line, which is what log shippers,
3
- * dataset files and streaming APIs emit.
4
- *
5
- * Reading is tolerant where tolerance is safe — a trailing newline, blank
6
- * lines, CRLF endings, and a byte-order mark are all fine — and strict where
7
- * it is not: a line that does not parse is reported with its 1-based line
8
- * number and the parser's message, never dropped in silence.
9
- */
10
1
  export type JsonlRecord = {
11
2
  line: number;
12
3
  value: unknown;
@@ -33,7 +24,7 @@ export declare function parseJsonl(text: string, maxRecords: number, stopOnError
33
24
  * format relies on. `undefined` entries are skipped and counted, since
34
25
  * `JSON.stringify(undefined)` is not a value at all.
35
26
  */
36
- export declare function writeJsonl(values: ReadonlyArray<unknown>, trailingNewline: boolean): {
27
+ export declare function writeJsonl(values: ReadonlyArray<unknown>, trailingNewline: boolean, maxChars?: number): {
37
28
  text: string;
38
29
  skipped: number;
39
30
  };
package/dist/lib/jsonl.js CHANGED
@@ -7,6 +7,7 @@
7
7
  * it is not: a line that does not parse is reported with its 1-based line
8
8
  * number and the parser's message, never dropped in silence.
9
9
  */
10
+ import { MAX_NESTING_DEPTH, OutputLimitError, jsonTextLength, jsonTextNestsDeeper } from "./json";
10
11
  /**
11
12
  * Parse JSONL. `maxRecords` bounds the result; `stopOnError` makes the first
12
13
  * bad line end the parse, which is what a caller wants when the file is a
@@ -27,6 +28,10 @@ export function parseJsonl(text, maxRecords, stopOnError) {
27
28
  break;
28
29
  }
29
30
  try {
31
+ // Checked before parsing, like every JSON document this package reads.
32
+ if (jsonTextNestsDeeper(raw, MAX_NESTING_DEPTH)) {
33
+ throw new Error(`nests deeper than ${MAX_NESTING_DEPTH} levels`);
34
+ }
30
35
  records.push({ line: i + 1, value: JSON.parse(raw) });
31
36
  }
32
37
  catch (err) {
@@ -47,15 +52,24 @@ export function parseJsonl(text, maxRecords, stopOnError) {
47
52
  * format relies on. `undefined` entries are skipped and counted, since
48
53
  * `JSON.stringify(undefined)` is not a value at all.
49
54
  */
50
- export function writeJsonl(values, trailingNewline) {
55
+ export function writeJsonl(values, trailingNewline, maxChars = Number.POSITIVE_INFINITY) {
51
56
  const lines = [];
52
57
  let skipped = 0;
58
+ let chars = 0;
53
59
  for (const value of values) {
60
+ // Measured before it is built: a record converted from CSV repeats every
61
+ // column name, so one line can be far larger than its source row.
62
+ if (maxChars !== Number.POSITIVE_INFINITY) {
63
+ const size = jsonTextLength(value, 0, maxChars - chars);
64
+ if (chars + size + 1 > maxChars)
65
+ throw new OutputLimitError(maxChars, "the JSONL");
66
+ }
54
67
  const text = JSON.stringify(value);
55
68
  if (text === undefined) {
56
69
  skipped += 1;
57
70
  continue;
58
71
  }
72
+ chars += text.length + 1;
59
73
  lines.push(text);
60
74
  }
61
75
  const body = lines.join("\n");
@@ -33,6 +33,7 @@
33
33
  * `&&`. Anything unsupported raises a parse error naming the offending
34
34
  * position, never a silent empty result.
35
35
  */
36
+ import { type RegexAnswers } from "@crewhaus/tool-schema";
36
37
  export type PathStep = {
37
38
  kind: "child";
38
39
  name: string;
@@ -72,11 +73,46 @@ export type PathMatch = {
72
73
  * back as an array index and matches nothing.
73
74
  */
74
75
  export type PathSegment = string | number;
75
- /** Run a parsed path over a value, returning matches in document order. */
76
- export declare function queryPath(root: unknown, steps: ReadonlyArray<PathStep>, limit: number): {
76
+ /**
77
+ * The most nodes one query may visit, counting every node a step produces
78
+ * and every node a `..` walk passes. A single pass over the largest document
79
+ * this package accepts is well inside it; a path that walks the document
80
+ * again for each node it found (`$..*..x` over a deep document) is what it
81
+ * stops.
82
+ */
83
+ export declare const MAX_QUERY_VISITS = 5000000;
84
+ export type QueryResult = {
77
85
  matches: PathMatch[];
86
+ /**
87
+ * True when there are, or may be, matches past the ones returned: the
88
+ * limit was reached, or the query stopped at the visit budget.
89
+ */
78
90
  truncated: boolean;
91
+ /** Set when the query stopped at MAX_QUERY_VISITS rather than at the limit. */
92
+ stoppedAtVisits?: number;
93
+ /**
94
+ * Nodes a `=~` filter could not be answered on (the regex worker's
95
+ * deadline, the engine giving up): left out, neither matched nor ruled
96
+ * out, with their paths. Absent when there were none.
97
+ */
98
+ undetermined?: {
99
+ count: number;
100
+ paths: string[];
101
+ };
79
102
  };
103
+ /**
104
+ * Run a parsed path over a value, returning matches in document order.
105
+ *
106
+ * Evaluated lazily, depth first: each node a step produces runs through the
107
+ * remaining steps before the next is produced, which gives exactly the
108
+ * step-by-step, document-order result, and lets the query stop as soon as it
109
+ * has one match more than `limit` (to know it truncated). 0.7.0 built each
110
+ * step's whole node list first and then cut it at 20 x limit — so `$..*`
111
+ * materialized every node with its path, and a cut in the middle of a path
112
+ * silently dropped real matches: `$[*].b` over thirty records with `b` only
113
+ * in the last answered zero matches, not truncated.
114
+ */
115
+ export declare function queryPath(root: unknown, steps: ReadonlyArray<PathStep>, limit: number, maxVisits?: number, regex?: RegexAnswers): QueryResult;
80
116
  /** Python slice semantics, including negative bounds and a negative step. */
81
117
  export declare function sliceIndices(length: number, start: number | null, end: number | null, step: number): number[];
82
118
  /**