@crewhaus/tool-data 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +9 -0
- package/README.md +38 -1
- package/dist/index.js +280 -63
- package/dist/lib/csv.d.ts +27 -2
- package/dist/lib/csv.js +77 -14
- package/dist/lib/diff.js +3 -3
- package/dist/lib/json.d.ts +51 -0
- package/dist/lib/json.js +174 -2
- package/dist/lib/jsonl.d.ts +1 -10
- package/dist/lib/jsonl.js +15 -1
- package/dist/lib/jsonpath.d.ts +38 -2
- package/dist/lib/jsonpath.js +166 -53
- package/dist/lib/own-record.d.ts +22 -0
- package/dist/lib/own-record.js +66 -0
- package/dist/lib/patch.d.ts +18 -2
- package/dist/lib/patch.js +72 -21
- package/dist/lib/table.d.ts +45 -8
- package/dist/lib/table.js +164 -59
- package/dist/lib/toml.d.ts +8 -1
- package/dist/lib/toml.js +77 -48
- package/dist/lib/xml.js +23 -10
- package/dist/lib/yaml.d.ts +19 -2
- package/dist/lib/yaml.js +88 -42
- package/package.json +11 -6
package/dist/lib/csv.d.ts
CHANGED
|
@@ -31,6 +31,20 @@ export declare class CsvError extends Error {
|
|
|
31
31
|
readonly line: number;
|
|
32
32
|
constructor(message: string, line: number);
|
|
33
33
|
}
|
|
34
|
+
/**
|
|
35
|
+
* Why a delimiter/quote pair cannot describe a CSV file, or null when it can.
|
|
36
|
+
*
|
|
37
|
+
* One check for the reader and the writer, so neither accepts a dialect the
|
|
38
|
+
* other cannot honour. A line break cannot be the delimiter or the quote:
|
|
39
|
+
* rows are separated by line breaks, so a `\n` delimiter turns every field
|
|
40
|
+
* into a row (a writer's output read back as one header line of values and
|
|
41
|
+
* zero records), and a `\r` one splits CRLF rows into phantom columns. The
|
|
42
|
+
* quote character cannot be the delimiter, or a quoted field cannot be told
|
|
43
|
+
* from a field boundary. U+FEFF cannot be either: a reader drops it from the
|
|
44
|
+
* start of a file as a byte-order mark, so `["", "x"]` written with it as
|
|
45
|
+
* the delimiter read back as `["x"]`.
|
|
46
|
+
*/
|
|
47
|
+
export declare function csvDialectError(delimiter: string, quote: string): string | null;
|
|
34
48
|
/** Split CSV text into rows of raw string fields. Throws `CsvError` on an unterminated quote. */
|
|
35
49
|
export declare function parseCsv(text: string, options: CsvParseOptions): CsvParseResult;
|
|
36
50
|
/**
|
|
@@ -65,6 +79,13 @@ export type CsvWriteOptions = {
|
|
|
65
79
|
/** Quote every field, not only the ones that need it. */
|
|
66
80
|
quoteAll: boolean;
|
|
67
81
|
header: string[] | null;
|
|
82
|
+
/**
|
|
83
|
+
* Throw `OutputLimitError` once the text would pass this many characters.
|
|
84
|
+
* Records with keys that differ make a column for every key and a cell for
|
|
85
|
+
* every column in every row, so the output can be rows x columns even
|
|
86
|
+
* when the input was rows + columns.
|
|
87
|
+
*/
|
|
88
|
+
maxChars?: number;
|
|
68
89
|
};
|
|
69
90
|
/** Quote a single field if it contains the delimiter, a quote, or a line break. */
|
|
70
91
|
export declare function escapeCsvField(value: string, options: CsvWriteOptions): string;
|
|
@@ -74,7 +95,11 @@ export declare function escapeCsvField(value: string, options: CsvWriteOptions):
|
|
|
74
95
|
* `[object Object]` is how a pipeline loses data without noticing.
|
|
75
96
|
*/
|
|
76
97
|
export declare function cellToString(value: unknown): string;
|
|
77
|
-
/**
|
|
78
|
-
|
|
98
|
+
/**
|
|
99
|
+
* Serialize rows of already-stringified cells. Throws `CsvError` for a
|
|
100
|
+
* dialect `parseCsv` would refuse or misread (see `csvDialectError`), so
|
|
101
|
+
* nothing is written that this package cannot read back.
|
|
102
|
+
*/
|
|
103
|
+
export declare function writeCsvRows(rows: Iterable<ReadonlyArray<unknown>>, options: CsvWriteOptions): string;
|
|
79
104
|
/** Column order for a set of records: keys in first-seen order across every record. */
|
|
80
105
|
export declare function unionKeys(records: ReadonlyArray<Record<string, unknown>>): string[];
|
package/dist/lib/csv.js
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* caller's problem, and there is no support for backslash escapes, which
|
|
14
14
|
* RFC 4180 does not define.
|
|
15
15
|
*/
|
|
16
|
+
import { OutputLimitError, setOwn } from "./json";
|
|
16
17
|
export class CsvError extends Error {
|
|
17
18
|
line;
|
|
18
19
|
constructor(message, line) {
|
|
@@ -20,15 +21,43 @@ export class CsvError extends Error {
|
|
|
20
21
|
this.line = line;
|
|
21
22
|
}
|
|
22
23
|
}
|
|
23
|
-
/**
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
/**
|
|
25
|
+
* Why a delimiter/quote pair cannot describe a CSV file, or null when it can.
|
|
26
|
+
*
|
|
27
|
+
* One check for the reader and the writer, so neither accepts a dialect the
|
|
28
|
+
* other cannot honour. A line break cannot be the delimiter or the quote:
|
|
29
|
+
* rows are separated by line breaks, so a `\n` delimiter turns every field
|
|
30
|
+
* into a row (a writer's output read back as one header line of values and
|
|
31
|
+
* zero records), and a `\r` one splits CRLF rows into phantom columns. The
|
|
32
|
+
* quote character cannot be the delimiter, or a quoted field cannot be told
|
|
33
|
+
* from a field boundary. U+FEFF cannot be either: a reader drops it from the
|
|
34
|
+
* start of a file as a byte-order mark, so `["", "x"]` written with it as
|
|
35
|
+
* the delimiter read back as `["x"]`.
|
|
36
|
+
*/
|
|
37
|
+
export function csvDialectError(delimiter, quote) {
|
|
26
38
|
if (delimiter.length !== 1)
|
|
27
|
-
|
|
39
|
+
return "delimiter must be a single character";
|
|
28
40
|
if (quote.length !== 1)
|
|
29
|
-
|
|
41
|
+
return "quote must be a single character";
|
|
42
|
+
if (delimiter === "\ufeff" || quote === "\ufeff") {
|
|
43
|
+
return "neither delimiter nor quote can be U+FEFF — a reader drops it from the start of a file as a byte-order mark";
|
|
44
|
+
}
|
|
45
|
+
if (delimiter === "\r" || delimiter === "\n") {
|
|
46
|
+
return "delimiter cannot be a line break (CR or LF) — rows are separated by line breaks";
|
|
47
|
+
}
|
|
48
|
+
if (quote === "\r" || quote === "\n") {
|
|
49
|
+
return "quote cannot be a line break (CR or LF) — rows are separated by line breaks";
|
|
50
|
+
}
|
|
30
51
|
if (delimiter === quote)
|
|
31
|
-
|
|
52
|
+
return "delimiter and quote must differ";
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
/** Split CSV text into rows of raw string fields. Throws `CsvError` on an unterminated quote. */
|
|
56
|
+
export function parseCsv(text, options) {
|
|
57
|
+
const { delimiter, quote } = options;
|
|
58
|
+
const dialect = csvDialectError(delimiter, quote);
|
|
59
|
+
if (dialect !== null)
|
|
60
|
+
throw new CsvError(dialect, 0);
|
|
32
61
|
const src = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
|
|
33
62
|
const rows = [];
|
|
34
63
|
let row = [];
|
|
@@ -154,10 +183,10 @@ export function rowsToRecords(rows, header, infer, nullTokens) {
|
|
|
154
183
|
header.forEach((name, j) => {
|
|
155
184
|
const cell = row[j];
|
|
156
185
|
if (cell === undefined) {
|
|
157
|
-
rec
|
|
186
|
+
setOwn(rec, name, null);
|
|
158
187
|
return;
|
|
159
188
|
}
|
|
160
|
-
rec
|
|
189
|
+
setOwn(rec, name, infer ? inferScalar(cell, nullTokens) : cell);
|
|
161
190
|
});
|
|
162
191
|
records.push(rec);
|
|
163
192
|
});
|
|
@@ -211,16 +240,50 @@ export function cellToString(value) {
|
|
|
211
240
|
return value;
|
|
212
241
|
if (typeof value === "number" || typeof value === "boolean")
|
|
213
242
|
return String(value);
|
|
214
|
-
|
|
243
|
+
// JSON.stringify answers undefined for a function or a symbol, which no
|
|
244
|
+
// JSON value is; an empty field beats a crash in the quoting step.
|
|
245
|
+
return JSON.stringify(value) ?? "";
|
|
215
246
|
}
|
|
216
|
-
/**
|
|
247
|
+
/**
|
|
248
|
+
* Serialize rows of already-stringified cells. Throws `CsvError` for a
|
|
249
|
+
* dialect `parseCsv` would refuse or misread (see `csvDialectError`), so
|
|
250
|
+
* nothing is written that this package cannot read back.
|
|
251
|
+
*/
|
|
217
252
|
export function writeCsvRows(rows, options) {
|
|
253
|
+
const dialect = csvDialectError(options.delimiter, options.quote);
|
|
254
|
+
if (dialect !== null)
|
|
255
|
+
throw new CsvError(dialect, 0);
|
|
256
|
+
const max = options.maxChars ?? Number.POSITIVE_INFINITY;
|
|
218
257
|
const lines = [];
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
258
|
+
let chars = -options.newline.length;
|
|
259
|
+
const push = (line) => {
|
|
260
|
+
chars += line.length + options.newline.length;
|
|
261
|
+
if (chars > max)
|
|
262
|
+
throw new OutputLimitError(max, "the CSV");
|
|
263
|
+
lines.push(line);
|
|
264
|
+
};
|
|
265
|
+
const q = options.quote;
|
|
266
|
+
// Two cases a reader would otherwise misread, so both are quoted: a row
|
|
267
|
+
// whose only field is empty is a blank line, which a reader skips, and a
|
|
268
|
+
// leading U+FEFF in the file's first field is taken for a byte-order mark
|
|
269
|
+
// and dropped.
|
|
270
|
+
const encode = (cells) => {
|
|
271
|
+
if (cells.length === 1 && cells[0] === "")
|
|
272
|
+
return q + q;
|
|
273
|
+
return cells
|
|
274
|
+
.map((cell, i) => i === 0 && lines.length === 0 && cell.startsWith("\ufeff")
|
|
275
|
+
? q + cell.split(q).join(q + q) + q
|
|
276
|
+
: escapeCsvField(cell, options))
|
|
277
|
+
.join(options.delimiter);
|
|
278
|
+
};
|
|
279
|
+
if (options.header !== null)
|
|
280
|
+
push(encode(options.header));
|
|
222
281
|
for (const row of rows) {
|
|
223
|
-
|
|
282
|
+
// A row's cells are delimiter-separated at the least; checked before the
|
|
283
|
+
// row is built, so one enormous row is not built to be refused.
|
|
284
|
+
if (chars + row.length > max)
|
|
285
|
+
throw new OutputLimitError(max, "the CSV");
|
|
286
|
+
push(encode(row.map(cellToString)));
|
|
224
287
|
}
|
|
225
288
|
return lines.join(options.newline);
|
|
226
289
|
}
|
package/dist/lib/diff.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* wrong one for a list of records — for records, key the arrays first with
|
|
8
8
|
* `keyArraysBy`, which matches elements by a field instead of by position.
|
|
9
9
|
*/
|
|
10
|
-
import { deepEqual, isPlainObject, typeOf } from "./json";
|
|
10
|
+
import { deepEqual, getOwn, isPlainObject, typeOf } from "./json";
|
|
11
11
|
import { formatPointer } from "./patch";
|
|
12
12
|
/** Compare two values, returning a flat list of differences by JSON Pointer path. */
|
|
13
13
|
export function deepDiff(a, b, options) {
|
|
@@ -119,7 +119,7 @@ function canKey(arr, key) {
|
|
|
119
119
|
for (const el of arr) {
|
|
120
120
|
if (!isPlainObject(el))
|
|
121
121
|
return false;
|
|
122
|
-
const v = el
|
|
122
|
+
const v = getOwn(el, key);
|
|
123
123
|
if (v === undefined || v === null || typeof v === "object")
|
|
124
124
|
return false;
|
|
125
125
|
const s = String(v);
|
|
@@ -133,7 +133,7 @@ function diffKeyedArrays(left, right, key, path, depth, recurse, push) {
|
|
|
133
133
|
const index = (arr) => {
|
|
134
134
|
const m = new Map();
|
|
135
135
|
arr.forEach((el, i) => {
|
|
136
|
-
const v = el
|
|
136
|
+
const v = getOwn(el, key);
|
|
137
137
|
m.set(String(v), { i, value: el });
|
|
138
138
|
});
|
|
139
139
|
return m;
|
package/dist/lib/json.d.ts
CHANGED
|
@@ -11,6 +11,24 @@ export type JsonValue = null | boolean | number | string | JsonValue[] | {
|
|
|
11
11
|
};
|
|
12
12
|
/** A plain object, as distinct from an array or a primitive. */
|
|
13
13
|
export declare function isPlainObject(value: unknown): value is Record<string, unknown>;
|
|
14
|
+
/**
|
|
15
|
+
* Read `key` only when it is the object's OWN property. A key taken from the
|
|
16
|
+
* data (a dotted path segment, a TOML table name, an XML element name) must
|
|
17
|
+
* never reach an inherited member: `constructor` would read
|
|
18
|
+
* `Object.prototype.constructor`, and `__proto__` would hand back
|
|
19
|
+
* `Object.prototype` itself as something to write into — which is how one
|
|
20
|
+
* hostile document polluted every object in the process.
|
|
21
|
+
*/
|
|
22
|
+
export declare function getOwn(obj: Record<string, unknown>, key: string): unknown;
|
|
23
|
+
/**
|
|
24
|
+
* Write `key` as an own data property. For every key but `__proto__` plain
|
|
25
|
+
* assignment already does that; `__proto__` alone is an accessor inherited
|
|
26
|
+
* from `Object.prototype`, whose setter would replace the object's prototype
|
|
27
|
+
* (dropping the key) instead of storing it. `defineProperty` stores it as the
|
|
28
|
+
* data it is, so `{"__proto__": …}` round-trips like any other key — the
|
|
29
|
+
* way `JSON.parse` itself reads it.
|
|
30
|
+
*/
|
|
31
|
+
export declare function setOwn(obj: Record<string, unknown>, key: string, value: unknown): void;
|
|
14
32
|
/** Structural equality over JSON values. Key order is irrelevant; NaN never appears in JSON. */
|
|
15
33
|
export declare function deepEqual(a: unknown, b: unknown): boolean;
|
|
16
34
|
/** A structured clone that only has to handle JSON shapes, so no cycles exist. */
|
|
@@ -63,3 +81,36 @@ export declare function parseJson(text: string): {
|
|
|
63
81
|
};
|
|
64
82
|
/** A short, human-readable type name used in diffs and error messages. */
|
|
65
83
|
export declare function typeOf(value: unknown): string;
|
|
84
|
+
/**
|
|
85
|
+
* The deepest nesting a document may have: each `[` or `{` is one level, a
|
|
86
|
+
* scalar document is depth 0. XmlParse's own ceiling, and far past any real
|
|
87
|
+
* configuration, API response or AST dump.
|
|
88
|
+
*/
|
|
89
|
+
export declare const MAX_NESTING_DEPTH = 256;
|
|
90
|
+
/**
|
|
91
|
+
* Whether JSON text nests deeper than `limit`, from one linear pass over the
|
|
92
|
+
* characters: brackets inside string literals (escapes honoured) do not
|
|
93
|
+
* count, and nothing is parsed, so a hostile document is refused before a
|
|
94
|
+
* deep tree exists. Malformed text may be over-counted; it would fail to
|
|
95
|
+
* parse anyway.
|
|
96
|
+
*/
|
|
97
|
+
export declare function jsonTextNestsDeeper(text: string, limit: number): boolean;
|
|
98
|
+
/** Whether a parsed value nests deeper than `limit`. Iterative: no recursion to overflow. */
|
|
99
|
+
export declare function nestingDepthExceeds(value: unknown, limit: number): boolean;
|
|
100
|
+
/** A result, or a part of one, that would be larger than its tool may return. */
|
|
101
|
+
export declare class OutputLimitError extends Error {
|
|
102
|
+
readonly limit: number;
|
|
103
|
+
constructor(limit: number, what?: string);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* The length of `JSON.stringify(value, null, indent)`, computed without
|
|
107
|
+
* building it, and abandoned as soon as it passes `cap` (the answer is then
|
|
108
|
+
* some number above `cap`). Iterative and exact for JSON values; a
|
|
109
|
+
* non-plain object (none of this package's readers make one) is measured by
|
|
110
|
+
* serializing it.
|
|
111
|
+
*
|
|
112
|
+
* Measured first so that a result too large to return is refused before it
|
|
113
|
+
* exists: pretty-printing multiplies every line by depth x indent, and a
|
|
114
|
+
* record set can repeat a long key once per row.
|
|
115
|
+
*/
|
|
116
|
+
export declare function jsonTextLength(value: unknown, indent: number, cap: number): number;
|
package/dist/lib/json.js
CHANGED
|
@@ -10,6 +10,38 @@
|
|
|
10
10
|
export function isPlainObject(value) {
|
|
11
11
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
12
12
|
}
|
|
13
|
+
/**
|
|
14
|
+
* Read `key` only when it is the object's OWN property. A key taken from the
|
|
15
|
+
* data (a dotted path segment, a TOML table name, an XML element name) must
|
|
16
|
+
* never reach an inherited member: `constructor` would read
|
|
17
|
+
* `Object.prototype.constructor`, and `__proto__` would hand back
|
|
18
|
+
* `Object.prototype` itself as something to write into — which is how one
|
|
19
|
+
* hostile document polluted every object in the process.
|
|
20
|
+
*/
|
|
21
|
+
export function getOwn(obj, key) {
|
|
22
|
+
return Object.hasOwn(obj, key) ? obj[key] : undefined;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Write `key` as an own data property. For every key but `__proto__` plain
|
|
26
|
+
* assignment already does that; `__proto__` alone is an accessor inherited
|
|
27
|
+
* from `Object.prototype`, whose setter would replace the object's prototype
|
|
28
|
+
* (dropping the key) instead of storing it. `defineProperty` stores it as the
|
|
29
|
+
* data it is, so `{"__proto__": …}` round-trips like any other key — the
|
|
30
|
+
* way `JSON.parse` itself reads it.
|
|
31
|
+
*/
|
|
32
|
+
export function setOwn(obj, key, value) {
|
|
33
|
+
if (key === "__proto__") {
|
|
34
|
+
Object.defineProperty(obj, key, {
|
|
35
|
+
value,
|
|
36
|
+
writable: true,
|
|
37
|
+
enumerable: true,
|
|
38
|
+
configurable: true,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
obj[key] = value;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
13
45
|
/** Structural equality over JSON values. Key order is irrelevant; NaN never appears in JSON. */
|
|
14
46
|
export function deepEqual(a, b) {
|
|
15
47
|
if (a === b)
|
|
@@ -45,7 +77,7 @@ export function deepClone(value) {
|
|
|
45
77
|
if (isPlainObject(value)) {
|
|
46
78
|
const out = {};
|
|
47
79
|
for (const k of Object.keys(value))
|
|
48
|
-
out
|
|
80
|
+
setOwn(out, k, deepClone(value[k]));
|
|
49
81
|
return out;
|
|
50
82
|
}
|
|
51
83
|
return value;
|
|
@@ -74,7 +106,7 @@ export function sortKeysDeep(value) {
|
|
|
74
106
|
if (isPlainObject(value)) {
|
|
75
107
|
const out = {};
|
|
76
108
|
for (const k of Object.keys(value).sort())
|
|
77
|
-
out
|
|
109
|
+
setOwn(out, k, sortKeysDeep(value[k]));
|
|
78
110
|
return out;
|
|
79
111
|
}
|
|
80
112
|
return value;
|
|
@@ -131,3 +163,143 @@ export function typeOf(value) {
|
|
|
131
163
|
return "array";
|
|
132
164
|
return typeof value;
|
|
133
165
|
}
|
|
166
|
+
// ---------------------------------------------------------------------------
|
|
167
|
+
// Bounds on nesting and on output.
|
|
168
|
+
//
|
|
169
|
+
// A document's cost here is not its length. Every tool that walks a value
|
|
170
|
+
// does work per level as well as per node — a merge that cloned the rest of
|
|
171
|
+
// the tree at each level, a query that copied the path at each level, a
|
|
172
|
+
// pretty-printer that writes depth x indent spaces on every line — so a
|
|
173
|
+
// 48 KB document nested 8,000 deep cost gigabytes. Depth is capped at the
|
|
174
|
+
// door, and every result is measured before it is built.
|
|
175
|
+
/**
|
|
176
|
+
* The deepest nesting a document may have: each `[` or `{` is one level, a
|
|
177
|
+
* scalar document is depth 0. XmlParse's own ceiling, and far past any real
|
|
178
|
+
* configuration, API response or AST dump.
|
|
179
|
+
*/
|
|
180
|
+
export const MAX_NESTING_DEPTH = 256;
|
|
181
|
+
/**
|
|
182
|
+
* Whether JSON text nests deeper than `limit`, from one linear pass over the
|
|
183
|
+
* characters: brackets inside string literals (escapes honoured) do not
|
|
184
|
+
* count, and nothing is parsed, so a hostile document is refused before a
|
|
185
|
+
* deep tree exists. Malformed text may be over-counted; it would fail to
|
|
186
|
+
* parse anyway.
|
|
187
|
+
*/
|
|
188
|
+
export function jsonTextNestsDeeper(text, limit) {
|
|
189
|
+
let depth = 0;
|
|
190
|
+
let inString = false;
|
|
191
|
+
for (let i = 0; i < text.length; i++) {
|
|
192
|
+
const c = text.charCodeAt(i);
|
|
193
|
+
if (inString) {
|
|
194
|
+
// A backslash escapes the character after it.
|
|
195
|
+
if (c === 0x5c)
|
|
196
|
+
i += 1;
|
|
197
|
+
else if (c === 0x22)
|
|
198
|
+
inString = false;
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
if (c === 0x22)
|
|
202
|
+
inString = true;
|
|
203
|
+
else if (c === 0x5b || c === 0x7b) {
|
|
204
|
+
depth += 1;
|
|
205
|
+
if (depth > limit)
|
|
206
|
+
return true;
|
|
207
|
+
}
|
|
208
|
+
else if (c === 0x5d || c === 0x7d) {
|
|
209
|
+
if (depth > 0)
|
|
210
|
+
depth -= 1;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
return false;
|
|
214
|
+
}
|
|
215
|
+
/** Whether a parsed value nests deeper than `limit`. Iterative: no recursion to overflow. */
|
|
216
|
+
export function nestingDepthExceeds(value, limit) {
|
|
217
|
+
const stack = [[value, 0]];
|
|
218
|
+
while (stack.length > 0) {
|
|
219
|
+
const [node, depth] = stack.pop();
|
|
220
|
+
const children = Array.isArray(node) ? node : isPlainObject(node) ? Object.values(node) : null;
|
|
221
|
+
if (children === null)
|
|
222
|
+
continue;
|
|
223
|
+
if (depth + 1 > limit)
|
|
224
|
+
return true;
|
|
225
|
+
for (const child of children) {
|
|
226
|
+
if (typeof child === "object" && child !== null)
|
|
227
|
+
stack.push([child, depth + 1]);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return false;
|
|
231
|
+
}
|
|
232
|
+
/** A result, or a part of one, that would be larger than its tool may return. */
|
|
233
|
+
export class OutputLimitError extends Error {
|
|
234
|
+
limit;
|
|
235
|
+
constructor(limit, what = "the result") {
|
|
236
|
+
super(`${what} would be more than ${limit} characters`);
|
|
237
|
+
this.limit = limit;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
/** Nothing a value's JSON form renders as: skipped in an object, `null` in an array. */
|
|
241
|
+
const renders = (v) => v !== undefined && typeof v !== "function" && typeof v !== "symbol";
|
|
242
|
+
/**
|
|
243
|
+
* The length of `JSON.stringify(value, null, indent)`, computed without
|
|
244
|
+
* building it, and abandoned as soon as it passes `cap` (the answer is then
|
|
245
|
+
* some number above `cap`). Iterative and exact for JSON values; a
|
|
246
|
+
* non-plain object (none of this package's readers make one) is measured by
|
|
247
|
+
* serializing it.
|
|
248
|
+
*
|
|
249
|
+
* Measured first so that a result too large to return is refused before it
|
|
250
|
+
* exists: pretty-printing multiplies every line by depth x indent, and a
|
|
251
|
+
* record set can repeat a long key once per row.
|
|
252
|
+
*/
|
|
253
|
+
export function jsonTextLength(value, indent, cap) {
|
|
254
|
+
if (!renders(value))
|
|
255
|
+
return 0;
|
|
256
|
+
const k = Math.min(10, Math.max(0, Math.floor(indent)));
|
|
257
|
+
let total = 0;
|
|
258
|
+
const stack = [[value, 0]];
|
|
259
|
+
while (stack.length > 0) {
|
|
260
|
+
const [node, depth] = stack.pop();
|
|
261
|
+
if (node === null || !renders(node)) {
|
|
262
|
+
total += 4; // null, or a non-rendering array element, which becomes null
|
|
263
|
+
}
|
|
264
|
+
else if (typeof node === "string") {
|
|
265
|
+
total += JSON.stringify(node).length;
|
|
266
|
+
}
|
|
267
|
+
else if (typeof node === "number") {
|
|
268
|
+
total += Number.isFinite(node) ? String(node).length : 4;
|
|
269
|
+
}
|
|
270
|
+
else if (typeof node === "boolean") {
|
|
271
|
+
total += node ? 4 : 5;
|
|
272
|
+
}
|
|
273
|
+
else if (Array.isArray(node)) {
|
|
274
|
+
const n = node.length;
|
|
275
|
+
if (n === 0)
|
|
276
|
+
total += 2;
|
|
277
|
+
else {
|
|
278
|
+
total += 2 + (n - 1) + (k === 0 ? 0 : n * (1 + (depth + 1) * k) + 1 + depth * k);
|
|
279
|
+
for (let i = n - 1; i >= 0; i--)
|
|
280
|
+
stack.push([node[i], depth + 1]);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
else if (isPlainObject(node)) {
|
|
284
|
+
let m = 0;
|
|
285
|
+
for (const key of Object.keys(node)) {
|
|
286
|
+
const v = node[key];
|
|
287
|
+
if (!renders(v))
|
|
288
|
+
continue;
|
|
289
|
+
m += 1;
|
|
290
|
+
total += JSON.stringify(key).length + (k === 0 ? 1 : 2);
|
|
291
|
+
stack.push([v, depth + 1]);
|
|
292
|
+
}
|
|
293
|
+
if (m === 0)
|
|
294
|
+
total += 2;
|
|
295
|
+
else
|
|
296
|
+
total += 2 + (m - 1) + (k === 0 ? 0 : m * (1 + (depth + 1) * k) + 1 + depth * k);
|
|
297
|
+
}
|
|
298
|
+
else {
|
|
299
|
+
total += (JSON.stringify(node) ?? "null").length;
|
|
300
|
+
}
|
|
301
|
+
if (total > cap)
|
|
302
|
+
return total;
|
|
303
|
+
}
|
|
304
|
+
return total;
|
|
305
|
+
}
|
package/dist/lib/jsonl.d.ts
CHANGED
|
@@ -1,12 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Line-delimited JSON. One value per line, which is what log shippers,
|
|
3
|
-
* dataset files and streaming APIs emit.
|
|
4
|
-
*
|
|
5
|
-
* Reading is tolerant where tolerance is safe — a trailing newline, blank
|
|
6
|
-
* lines, CRLF endings, and a byte-order mark are all fine — and strict where
|
|
7
|
-
* it is not: a line that does not parse is reported with its 1-based line
|
|
8
|
-
* number and the parser's message, never dropped in silence.
|
|
9
|
-
*/
|
|
10
1
|
export type JsonlRecord = {
|
|
11
2
|
line: number;
|
|
12
3
|
value: unknown;
|
|
@@ -33,7 +24,7 @@ export declare function parseJsonl(text: string, maxRecords: number, stopOnError
|
|
|
33
24
|
* format relies on. `undefined` entries are skipped and counted, since
|
|
34
25
|
* `JSON.stringify(undefined)` is not a value at all.
|
|
35
26
|
*/
|
|
36
|
-
export declare function writeJsonl(values: ReadonlyArray<unknown>, trailingNewline: boolean): {
|
|
27
|
+
export declare function writeJsonl(values: ReadonlyArray<unknown>, trailingNewline: boolean, maxChars?: number): {
|
|
37
28
|
text: string;
|
|
38
29
|
skipped: number;
|
|
39
30
|
};
|
package/dist/lib/jsonl.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* it is not: a line that does not parse is reported with its 1-based line
|
|
8
8
|
* number and the parser's message, never dropped in silence.
|
|
9
9
|
*/
|
|
10
|
+
import { MAX_NESTING_DEPTH, OutputLimitError, jsonTextLength, jsonTextNestsDeeper } from "./json";
|
|
10
11
|
/**
|
|
11
12
|
* Parse JSONL. `maxRecords` bounds the result; `stopOnError` makes the first
|
|
12
13
|
* bad line end the parse, which is what a caller wants when the file is a
|
|
@@ -27,6 +28,10 @@ export function parseJsonl(text, maxRecords, stopOnError) {
|
|
|
27
28
|
break;
|
|
28
29
|
}
|
|
29
30
|
try {
|
|
31
|
+
// Checked before parsing, like every JSON document this package reads.
|
|
32
|
+
if (jsonTextNestsDeeper(raw, MAX_NESTING_DEPTH)) {
|
|
33
|
+
throw new Error(`nests deeper than ${MAX_NESTING_DEPTH} levels`);
|
|
34
|
+
}
|
|
30
35
|
records.push({ line: i + 1, value: JSON.parse(raw) });
|
|
31
36
|
}
|
|
32
37
|
catch (err) {
|
|
@@ -47,15 +52,24 @@ export function parseJsonl(text, maxRecords, stopOnError) {
|
|
|
47
52
|
* format relies on. `undefined` entries are skipped and counted, since
|
|
48
53
|
* `JSON.stringify(undefined)` is not a value at all.
|
|
49
54
|
*/
|
|
50
|
-
export function writeJsonl(values, trailingNewline) {
|
|
55
|
+
export function writeJsonl(values, trailingNewline, maxChars = Number.POSITIVE_INFINITY) {
|
|
51
56
|
const lines = [];
|
|
52
57
|
let skipped = 0;
|
|
58
|
+
let chars = 0;
|
|
53
59
|
for (const value of values) {
|
|
60
|
+
// Measured before it is built: a record converted from CSV repeats every
|
|
61
|
+
// column name, so one line can be far larger than its source row.
|
|
62
|
+
if (maxChars !== Number.POSITIVE_INFINITY) {
|
|
63
|
+
const size = jsonTextLength(value, 0, maxChars - chars);
|
|
64
|
+
if (chars + size + 1 > maxChars)
|
|
65
|
+
throw new OutputLimitError(maxChars, "the JSONL");
|
|
66
|
+
}
|
|
54
67
|
const text = JSON.stringify(value);
|
|
55
68
|
if (text === undefined) {
|
|
56
69
|
skipped += 1;
|
|
57
70
|
continue;
|
|
58
71
|
}
|
|
72
|
+
chars += text.length + 1;
|
|
59
73
|
lines.push(text);
|
|
60
74
|
}
|
|
61
75
|
const body = lines.join("\n");
|
package/dist/lib/jsonpath.d.ts
CHANGED
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
* `&&`. Anything unsupported raises a parse error naming the offending
|
|
34
34
|
* position, never a silent empty result.
|
|
35
35
|
*/
|
|
36
|
+
import { type RegexAnswers } from "@crewhaus/tool-schema";
|
|
36
37
|
export type PathStep = {
|
|
37
38
|
kind: "child";
|
|
38
39
|
name: string;
|
|
@@ -72,11 +73,46 @@ export type PathMatch = {
|
|
|
72
73
|
* back as an array index and matches nothing.
|
|
73
74
|
*/
|
|
74
75
|
export type PathSegment = string | number;
|
|
75
|
-
/**
|
|
76
|
-
|
|
76
|
+
/**
|
|
77
|
+
* The most nodes one query may visit, counting every node a step produces
|
|
78
|
+
* and every node a `..` walk passes. A single pass over the largest document
|
|
79
|
+
* this package accepts is well inside it; a path that walks the document
|
|
80
|
+
* again for each node it found (`$..*..x` over a deep document) is what it
|
|
81
|
+
* stops.
|
|
82
|
+
*/
|
|
83
|
+
export declare const MAX_QUERY_VISITS = 5000000;
|
|
84
|
+
export type QueryResult = {
|
|
77
85
|
matches: PathMatch[];
|
|
86
|
+
/**
|
|
87
|
+
* True when there are, or may be, matches past the ones returned: the
|
|
88
|
+
* limit was reached, or the query stopped at the visit budget.
|
|
89
|
+
*/
|
|
78
90
|
truncated: boolean;
|
|
91
|
+
/** Set when the query stopped at MAX_QUERY_VISITS rather than at the limit. */
|
|
92
|
+
stoppedAtVisits?: number;
|
|
93
|
+
/**
|
|
94
|
+
* Nodes a `=~` filter could not be answered on (the regex worker's
|
|
95
|
+
* deadline, the engine giving up): left out, neither matched nor ruled
|
|
96
|
+
* out, with their paths. Absent when there were none.
|
|
97
|
+
*/
|
|
98
|
+
undetermined?: {
|
|
99
|
+
count: number;
|
|
100
|
+
paths: string[];
|
|
101
|
+
};
|
|
79
102
|
};
|
|
103
|
+
/**
|
|
104
|
+
* Run a parsed path over a value, returning matches in document order.
|
|
105
|
+
*
|
|
106
|
+
* Evaluated lazily, depth first: each node a step produces runs through the
|
|
107
|
+
* remaining steps before the next is produced, which gives exactly the
|
|
108
|
+
* step-by-step, document-order result, and lets the query stop as soon as it
|
|
109
|
+
* has one match more than `limit` (to know it truncated). 0.7.0 built each
|
|
110
|
+
* step's whole node list first and then cut it at 20 x limit — so `$..*`
|
|
111
|
+
* materialized every node with its path, and a cut in the middle of a path
|
|
112
|
+
* silently dropped real matches: `$[*].b` over thirty records with `b` only
|
|
113
|
+
* in the last answered zero matches, not truncated.
|
|
114
|
+
*/
|
|
115
|
+
export declare function queryPath(root: unknown, steps: ReadonlyArray<PathStep>, limit: number, maxVisits?: number, regex?: RegexAnswers): QueryResult;
|
|
80
116
|
/** Python slice semantics, including negative bounds and a negative step. */
|
|
81
117
|
export declare function sliceIndices(length: number, start: number | null, end: number | null, step: number): number[];
|
|
82
118
|
/**
|