echelongraph-mcp 3.0.2 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +676 -5
- package/README.md +79 -84
- package/dist/http.js +77 -84
- package/dist/httpPolicy.js +206 -14
- package/dist/index.js +489 -49
- package/dist/inputBounds.js +38 -0
- package/dist/logLines.js +86 -0
- package/dist/prompts.js +122 -23
- package/dist/readBody.js +140 -0
- package/dist/requiredText.js +23 -3
- package/dist/resources.js +3 -3
- package/dist/runtime.js +66 -1
- package/dist/schemaRefusal.js +147 -0
- package/dist/textBudget.js +826 -76
- package/dist/tools/check_affected.js +19 -6
- package/dist/tools/check_sbom.js +223 -25
- package/dist/tools/cve_intel.js +145 -35
- package/dist/tools/cve_remediation.js +101 -19
- package/dist/tools/epss_history.js +41 -5
- package/dist/tools/match_batch.js +57 -10
- package/dist/tools/packageRecords.js +58 -0
- package/dist/tools/scan_manifest.js +13 -4
- package/dist/tools/vendor_advisories.js +27 -4
- package/docs/exposure-radar.md +4 -0
- package/docs/rate-limits.md +9 -5
- package/docs/structured-results.md +23 -3
- package/docs/tools.md +139 -6
- package/package.json +3 -3
- package/server.json +2 -2
package/dist/textBudget.js
CHANGED
|
@@ -32,9 +32,20 @@ import * as z from "zod";
|
|
|
32
32
|
// in "…", every list in it to its first `cap` entries); then, for a tool that names the
|
|
33
33
|
// lists beside its rows (`sides`: check_affected's excluded sample, check_sbom's
|
|
34
34
|
// not_sent_purls), the same level with those lists cut as it says; then each next level with
|
|
35
|
-
// them cut; and when no level fits,
|
|
36
|
-
//
|
|
37
|
-
//
|
|
35
|
+
// them cut; and when no level fits, the lists beside the rows that the tool names
|
|
36
|
+
// beforeRows (#3272: cve_remediation's fixed_branches, patches and fix_references, which its
|
|
37
|
+
// structured cut thins and leaves in the same order) are cut as each says, one after another,
|
|
38
|
+
// then (3.1.0's phase A review, `capped`: fixed_branches) each entry's own list is cut to its
|
|
39
|
+
// first c entries, the most that fit, every entry kept and marked as the structured cut
|
|
40
|
+
// marks it, and then they lose entries from the end, the last named first, the room left
|
|
41
|
+
// going back to c; then rows are left out of the
|
|
42
|
+
// text, those the tool names first, then from the end, those it names last (leaveOutLast,
|
|
43
|
+
// #2899: check_affected's KEV-listed and ransomware matches) only after every other, the note
|
|
44
|
+
// naming each of those left out. A list the tool names beside its rows is cut before any row
|
|
45
|
+
// leaves the text, and, unless it is named beforeRows, before any row field the first level
|
|
46
|
+
// keeps (a list it does not name is cut only in step 4).
|
|
47
|
+
// With outFirstBefore (#3269: check_sbom), the rows the tool names are cut to the leanest
|
|
48
|
+
// level and leave first, as many as must, before any other row is cut to the level it names;
|
|
38
49
|
// 4. else every list keeps its first n entries, n halving, and then strings are cut too.
|
|
39
50
|
// Whenever the text is not the JSON whole, the note ends with one sentence, starting TEXT CUT,
|
|
40
51
|
// that says what the first text block leaves out, where structuredContent.data carries all of it,
|
|
@@ -66,7 +77,17 @@ import * as z from "zod";
|
|
|
66
77
|
// each entry on its own terms (an EntryLevel, #3132: check_sbom's rows keep at most the same
|
|
67
78
|
// number of matches, the most that fit, so only the rows with more lose any), and a step named
|
|
68
79
|
// keepAll (#3139: cve_remediation's vendor_remediations) keeps every entry at its leanest level
|
|
69
|
-
// while the later steps are cut, its entries leaving only once they all are;
|
|
80
|
+
// while the later steps are cut, its entries leaving only once they all are; and a step named
|
|
81
|
+
// with outFirst (#3269: check_sbom's results) sends the entries leaveOutFirst names out, cut
|
|
82
|
+
// lean, before the others are cut past a given level, and has them back with the room left,
|
|
83
|
+
// but (#3269's review, outFirst.floor) keeps them while the others, capped anyway, fit at a
|
|
84
|
+
// floor; and a tool that pages names the page size at which a call of its first entries keeps
|
|
85
|
+
// them all (PageAt.firstFit), counted as this cut would cut that call; #3269's second review:
|
|
86
|
+
// with EntryLevel.floor and leaveAt's floor no entry's list goes below its floor, the entries
|
|
87
|
+
// leave whole instead, chosen by fit, those DataCutStep.leaveLast holds kept first, and the
|
|
88
|
+
// note names by id those of them that left (#3269's third review: those it holds first and
|
|
89
|
+
// apart, and, where the tool gives the sentence, the sizes of calls of them alone that keep
|
|
90
|
+
// every one, PageAt.againFit);
|
|
70
91
|
// 2. then every other list at the top level of data, the same way. A list inside an object of
|
|
71
92
|
// data (get_cve's remediation summary, with fix_status, #2841) is cut only where a DataCut
|
|
72
93
|
// names it, and none names get_cve's. Once a step fits, the room it leaves goes back to the
|
|
@@ -242,6 +263,7 @@ export const FIRST_TEXT_BLOCK = {
|
|
|
242
263
|
export const FIRST_TEXT_BLOCK_BESIDE_A_CUT = {
|
|
243
264
|
opens: (chars) => `TEXT CUT: the API's answer is ${chars} characters of JSON, more than the ${DATA_TEXT_BUDGET} the first text block holds, so that block is cut, and structuredContent.data carries the answer whole with full: true (DATA CUT, below).`,
|
|
244
265
|
leftOut: (n, which) => `the other ${n} are left out of it${which}, and structuredContent.data holds them with full: true`,
|
|
266
|
+
full: true,
|
|
245
267
|
};
|
|
246
268
|
/** The first text block for `data` within DATA_TEXT_BUDGET, and the TEXT CUT sentence when it is cut. */
|
|
247
269
|
export function dataText(data, cut, place = FIRST_TEXT_BLOCK) {
|
|
@@ -261,59 +283,271 @@ export function dataText(data, cut, place = FIRST_TEXT_BLOCK) {
|
|
|
261
283
|
if (cut && isPlainObject(data) && Array.isArray(rows) && rows.length > 0 && cut.levels.length > 0) {
|
|
262
284
|
const n = rows.length;
|
|
263
285
|
// The lists beside the rows, cut once; tried with the first level after it fails with them
|
|
264
|
-
// whole, and with every later level.
|
|
265
|
-
const
|
|
286
|
+
// whole, and with every later level. (Those named beforeRows, #3272, are cut only below.)
|
|
287
|
+
const early = (cut.sides ?? []).filter((side) => !side.beforeRows);
|
|
288
|
+
const sides = early.length ? cutSides(data, early) : { fields: {}, said: [] };
|
|
266
289
|
const sided = sides.said.length > 0;
|
|
267
290
|
const attempts = cut.levels.flatMap((level, i) => (i === 0 && sided ? [{ level, sided: false }, { level, sided: true }] : [{ level, sided }]));
|
|
291
|
+
// The order rows leave the text in: those the tool names first (by value, in its order, each
|
|
292
|
+
// from the end), then the rest from the end.
|
|
293
|
+
const firstOut = cut.leaveOutFirst;
|
|
294
|
+
const isNamed = (i) => firstOut !== undefined && firstOut.values.includes(String(field(rows[i], firstOut.field)));
|
|
295
|
+
const lastOut = cut.leaveOutLast;
|
|
296
|
+
const isLast = (i) => lastOut !== undefined && (lastOut.fields.some((f) => field(rows[i], f) === true) || lastOut.has?.(rows[i]) === true);
|
|
297
|
+
// What a row named last has, as the note says it: "kev_listed or ransomware true" (#2899), or
|
|
298
|
+
// the tool's own words (#3284: "a KEV-listed or ransomware match").
|
|
299
|
+
const lastHas = lastOut ? (lastOut.said ?? `${orListed(lastOut.fields)} true`) : "";
|
|
300
|
+
const lastWhose = lastOut ? (lastOut.said ? `with ${lastOut.said}` : `whose ${orListed(lastOut.fields)} is true`) : "";
|
|
301
|
+
const lastWithout = lastOut ? (lastOut.said ? `without ${lastOut.said}` : `whose ${orListed(lastOut.fields)} is not true`) : "";
|
|
302
|
+
const order = [];
|
|
303
|
+
for (const value of firstOut?.values ?? []) {
|
|
304
|
+
for (let i = n - 1; i >= 0; i--)
|
|
305
|
+
if (field(rows[i], firstOut?.field ?? "") === value)
|
|
306
|
+
order.push(i);
|
|
307
|
+
}
|
|
308
|
+
const named = order.length;
|
|
309
|
+
const queued = new Set(order);
|
|
310
|
+
for (let i = n - 1; i >= 0; i--)
|
|
311
|
+
if (!queued.has(i) && !isLast(i))
|
|
312
|
+
order.push(i);
|
|
313
|
+
for (let i = n - 1; i >= 0; i--)
|
|
314
|
+
if (!queued.has(i) && isLast(i))
|
|
315
|
+
order.push(i);
|
|
316
|
+
// Rows leave `cutRows` in that order, the first `upTo` of it at most, until the text fits; the
|
|
317
|
+
// answer so cut, its sentences after those `said` gives for the number of named rows that left,
|
|
318
|
+
// or undefined where it does not fit.
|
|
319
|
+
const leaving = (cutRows, beside, said, upTo = n) => {
|
|
320
|
+
const lens = cutRows.map((r) => 4 + JSON.stringify(r).length);
|
|
321
|
+
const emptyLen = lined({ ...beside, [cut.rows]: [] }).length;
|
|
322
|
+
const textLen = (kept, sum) => (kept === 0 ? emptyLen : emptyLen + 2 + sum + 2 * kept);
|
|
323
|
+
const out = new Set();
|
|
324
|
+
let sum = lens.reduce((t, x) => t + x, 0);
|
|
325
|
+
for (const i of order.slice(0, upTo)) {
|
|
326
|
+
if (textLen(n - out.size, sum) <= DATA_TEXT_BUDGET)
|
|
327
|
+
break;
|
|
328
|
+
out.add(i);
|
|
329
|
+
sum -= lens[i];
|
|
330
|
+
}
|
|
331
|
+
const kept = cutRows.filter((_, i) => !out.has(i));
|
|
332
|
+
const shown = lined({ ...beside, [cut.rows]: kept });
|
|
333
|
+
if (kept.length === 0 || shown.length > DATA_TEXT_BUDGET)
|
|
334
|
+
return undefined;
|
|
335
|
+
const outFirst = [...out].filter(isNamed).length;
|
|
336
|
+
// #2899: the rows left out are the last in order unless a row named last stayed behind them.
|
|
337
|
+
const inOrder = [...out].every((i) => i >= n - out.size);
|
|
338
|
+
// #3284: the same of the rows left out that the tool does not name first: the last of the
|
|
339
|
+
// rest unless a row named last stayed behind them.
|
|
340
|
+
const restOut = [...out].filter((i) => !isNamed(i));
|
|
341
|
+
const restIdx = [...Array(n).keys()].filter((i) => !isNamed(i));
|
|
342
|
+
const restInOrder = restOut.every((i) => restIdx.indexOf(i) >= restIdx.length - restOut.length);
|
|
343
|
+
const butLast = lastOut ? ` but for any ${lastWhose}${lastOut.why ? ` (${lastOut.why})` : ""}, which leave it last` : "";
|
|
344
|
+
// #3284: the rest, where a row named last stayed behind them: the last of those without one,
|
|
345
|
+
// or, where rows named last left too, each without one and then the last of those with one
|
|
346
|
+
// (said short: the note counts against the structured result's budget).
|
|
347
|
+
const lastOutN = [...out].filter(isLast).length;
|
|
348
|
+
const restOf = (m) => restInOrder || !lastOut
|
|
349
|
+
? `, and the last ${m} of the rest`
|
|
350
|
+
: lastOutN === 0
|
|
351
|
+
? `, and the last ${m} of the rest ${lastWithout}`
|
|
352
|
+
: `, and ${m} of the rest: each ${lastWithout}, then the last ${lastOutN} ${lastWhose}`;
|
|
353
|
+
const which = outFirst > 0
|
|
354
|
+
? `: the ${outFirst} whose ${firstOut?.field} is ${(firstOut?.values ?? []).join(" or ")} (${firstOut?.why})${out.size > outFirst ? restOf(out.size - outFirst) : ""}`
|
|
355
|
+
: inOrder || !lastOut
|
|
356
|
+
? `, the last ${out.size} in order`
|
|
357
|
+
: !lastOut.said
|
|
358
|
+
? `, the last ${out.size} in order${butLast}`
|
|
359
|
+
: lastOutN === 0
|
|
360
|
+
? `, the last ${out.size} ${lastWithout}`
|
|
361
|
+
: `: each ${lastWithout}, then the last ${lastOutN} ${lastWhose}`;
|
|
362
|
+
const elsewhere = place.leftOut ? place.leftOut(out.size, which) : `the other ${out.size} are in structuredContent.data only${which}`;
|
|
363
|
+
const left = out.size > 0 ? `Only ${kept.length} of the ${n} rows of ${cut.rows} are in it; ${elsewhere}.` : "";
|
|
364
|
+
// #2899: each row named last that the text still leaves out, by name.
|
|
365
|
+
const lastLeft = [...out].filter(isLast).sort((a, b) => a - b);
|
|
366
|
+
const named_ = () => {
|
|
367
|
+
if (!lastOut?.runs)
|
|
368
|
+
return listed(lastLeft.map((i) => String(field(rows[i], lastOut?.name ?? "") ?? `${cut.rows}[${i}]`)));
|
|
369
|
+
// #3284: by their numbers, in runs, at most OTHERS_LEFT_LISTED parts.
|
|
370
|
+
const ids = lastLeft.map((i) => field(rows[i], lastOut.name)).map((v, j) => (typeof v === "number" ? v : lastLeft[j])).sort((a, b) => a - b);
|
|
371
|
+
const r = runsSaid(ids);
|
|
372
|
+
return `at ${lastOut.name} ${r.said}${r.more > 0 ? `, and ${r.more} more past ${lastOut.name} ${r.last}` : ""}`;
|
|
373
|
+
};
|
|
374
|
+
const lastSaid = lastOut && lastLeft.length ? `Of the rows left out of it, ${lastLeft.length} ${lastLeft.length === 1 ? "has" : "have"} ${lastHas}: ${named_()}.` : "";
|
|
375
|
+
return { text: shown, said: say(...said(outFirst, out), left, lastSaid, cut.whole, out.size > 0 ? cut.page?.(kept.length, n, data) : undefined) };
|
|
376
|
+
};
|
|
268
377
|
let cutRows = [];
|
|
269
378
|
let beside = data;
|
|
270
379
|
for (const attempt of attempts) {
|
|
271
380
|
const { level } = attempt;
|
|
381
|
+
beside = attempt.sided ? { ...data, ...sides.fields } : data;
|
|
382
|
+
sidesSaid = attempt.sided ? sides.said : [];
|
|
383
|
+
// #3269: before any row is cut to the level outFirstBefore names, the rows leaveOutFirst
|
|
384
|
+
// names are cut to the leanest level, the others kept at the level before, and then leave,
|
|
385
|
+
// as many as must. Where even that does not fit, the cut goes on as it did through 3.0.2:
|
|
386
|
+
// every row at the next level, then rows leaving, those named first.
|
|
387
|
+
const li = cut.levels.indexOf(level);
|
|
388
|
+
if (cut.outFirstBefore !== undefined && li === cut.outFirstBefore && li > 0 && named > 0 && attempt === attempts.find((a) => a.level === level)) {
|
|
389
|
+
const prev = cut.levels[li - 1];
|
|
390
|
+
const lean = cut.levels[cut.levels.length - 1];
|
|
391
|
+
const s = { left: new Set(), clipped: 0, capped: 0 };
|
|
392
|
+
const sn = { left: new Set(), clipped: 0, capped: 0 };
|
|
393
|
+
// #3269's review: the named rows whose strings the lean level clips, or whose lists it
|
|
394
|
+
// caps, by index, so the sentence says so of the ones kept (round 1 said only the fields
|
|
395
|
+
// they leave out, and a clean row's 150-character purl came back clipped unsaid).
|
|
396
|
+
const clippedAt = new Set();
|
|
397
|
+
const cappedAt = new Set();
|
|
398
|
+
const apart = rows.map((r, i) => {
|
|
399
|
+
if (!isPlainObject(r))
|
|
400
|
+
return cutValue(r, prev.clip, prev.cap ?? Number.POSITIVE_INFINITY, s);
|
|
401
|
+
if (!isNamed(i))
|
|
402
|
+
return cutRow(r, prev.keep, prev, s);
|
|
403
|
+
const [clipped, capped] = [sn.clipped, sn.capped];
|
|
404
|
+
const lean_ = cutRow(r, lean.keep, lean, sn);
|
|
405
|
+
if (sn.clipped > clipped)
|
|
406
|
+
clippedAt.add(i);
|
|
407
|
+
if (sn.capped > capped)
|
|
408
|
+
cappedAt.add(i);
|
|
409
|
+
return lean_;
|
|
410
|
+
});
|
|
411
|
+
const extra = [...sn.left].filter((f) => !s.left.has(f));
|
|
412
|
+
const apartSaid = (left, out) => {
|
|
413
|
+
const parts = [];
|
|
414
|
+
if (extra.length)
|
|
415
|
+
parts.push(`leave out ${listed(extra)} too`);
|
|
416
|
+
if ([...clippedAt].some((i) => !out.has(i)))
|
|
417
|
+
parts.push(`every string in each of them longer than ${lean.clip} characters keeps its first ${lean.clip}, ending in "…"`);
|
|
418
|
+
if (lean.cap !== undefined && [...cappedAt].some((i) => !out.has(i)))
|
|
419
|
+
parts.push(`every list in each of them longer than ${lean.cap} entries keeps its first ${lean.cap}`);
|
|
420
|
+
return named - left > 0 && parts.length ? `The ${named - left} rows of ${cut.rows} in it whose ${firstOut?.field} is ${(firstOut?.values ?? []).join(" or ")} ${parts.join("; ")}.` : "";
|
|
421
|
+
};
|
|
422
|
+
const prevSaid = levelSaid(cut.rows, n, prev, s);
|
|
423
|
+
const shown = lined({ ...beside, [cut.rows]: apart });
|
|
424
|
+
if (shown.length <= DATA_TEXT_BUDGET)
|
|
425
|
+
return { text: shown, said: say(prevSaid, apartSaid(0, new Set()), ...sidesSaid, cut.whole) };
|
|
426
|
+
// Only the named rows leave here.
|
|
427
|
+
const r = leaving(apart, beside, (left, out) => [prevSaid, apartSaid(left, out), ...sidesSaid], named);
|
|
428
|
+
if (r)
|
|
429
|
+
return r;
|
|
430
|
+
}
|
|
272
431
|
const s = { left: new Set(), clipped: 0, capped: 0 };
|
|
273
432
|
const cap = level.cap ?? Number.POSITIVE_INFINITY;
|
|
274
433
|
cutRows = rows.map((r) => (isPlainObject(r) ? cutRow(r, level.keep, level, s) : cutValue(r, level.clip, cap, s)));
|
|
275
434
|
rowsSaid = levelSaid(cut.rows, n, level, s);
|
|
276
|
-
beside = attempt.sided ? { ...data, ...sides.fields } : data;
|
|
277
|
-
sidesSaid = attempt.sided ? sides.said : [];
|
|
278
435
|
const shown = lined({ ...beside, [cut.rows]: cutRows });
|
|
279
436
|
if (shown.length <= DATA_TEXT_BUDGET)
|
|
280
437
|
return { text: shown, said: say(rowsSaid, ...sidesSaid, cut.whole) };
|
|
281
438
|
}
|
|
282
|
-
// No level fits:
|
|
283
|
-
//
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
const
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
439
|
+
// No level fits. #3272: the lists the tool names beforeRows, whole until now, are cut as each
|
|
440
|
+
// says, one after another in the order named, until the text fits with every row at the
|
|
441
|
+
// leanest level; then they lose entries, from the end, the last named first, each emptied
|
|
442
|
+
// before the next loses any (a DataCut's keepAll steps, in the same order); the note says
|
|
443
|
+
// what each keeps.
|
|
444
|
+
const spent = (cut.sides ?? []).filter((side) => side.beforeRows);
|
|
445
|
+
for (const side of spent) {
|
|
446
|
+
const one = cutSides(data, [side]);
|
|
447
|
+
if (!one.said.length)
|
|
448
|
+
continue;
|
|
449
|
+
beside = { ...beside, ...one.fields };
|
|
450
|
+
sidesSaid = [...sidesSaid, ...one.said];
|
|
451
|
+
const shown = lined({ ...beside, [cut.rows]: cutRows });
|
|
452
|
+
if (shown.length <= DATA_TEXT_BUDGET)
|
|
453
|
+
return { text: shown, said: say(rowsSaid, ...sidesSaid, cut.whole) };
|
|
293
454
|
}
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
const
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
455
|
+
// 3.1.0's phase A review: the lists named with `capped` have each entry's own list cut to its
|
|
456
|
+
// first c entries, the most that fit, before any entry of a list leaves (#3272's text emptied
|
|
457
|
+
// fixed_branches once one package of some 400 ranges did not fit alone, while 3.0.2's every-list
|
|
458
|
+
// cap kept 32 packages; data caps each package's ranges instead, and so does the text now).
|
|
459
|
+
const capping = spent.filter((side) => side.capped && Array.isArray(beside[side.list]) && beside[side.list].length > 0);
|
|
460
|
+
const caps = new Map();
|
|
461
|
+
const capList = (side, v, c) => v.map((e) => {
|
|
462
|
+
const inner = isPlainObject(e) ? e[side.capped.list] : undefined;
|
|
463
|
+
return isPlainObject(e) && Array.isArray(inner) && inner.length > c ? { ...e, [side.capped.list]: inner.slice(0, c), ...side.capped.mark(e, c) } : e;
|
|
464
|
+
});
|
|
465
|
+
const longest = (v, list) => v.reduce((m, e) => Math.max(m, Array.isArray(field(e, list)) ? field(e, list).length : 0), 0);
|
|
466
|
+
// The most c (from `least` up to the longest list less one) at which the text fits, or undefined.
|
|
467
|
+
const mostCap = (side, whole, with_) => {
|
|
468
|
+
const at = (c) => lined({ ...with_, [side.list]: capList(side, whole, c), [cut.rows]: cutRows }).length <= DATA_TEXT_BUDGET;
|
|
469
|
+
let lo = side.capped.least;
|
|
470
|
+
let hi = longest(whole, side.capped.list) - 1;
|
|
471
|
+
if (hi < lo || !at(lo))
|
|
472
|
+
return undefined;
|
|
473
|
+
while (lo < hi) {
|
|
474
|
+
const mid = Math.ceil((lo + hi) / 2);
|
|
475
|
+
if (at(mid))
|
|
476
|
+
lo = mid;
|
|
477
|
+
else
|
|
478
|
+
hi = mid - 1;
|
|
479
|
+
}
|
|
480
|
+
return lo;
|
|
481
|
+
};
|
|
482
|
+
const uncapped = new Map();
|
|
483
|
+
for (const side of capping) {
|
|
484
|
+
const whole = beside[side.list];
|
|
485
|
+
uncapped.set(side.list, whole);
|
|
486
|
+
const c = mostCap(side, whole, beside) ?? side.capped.least;
|
|
487
|
+
caps.set(side.list, c);
|
|
488
|
+
beside = { ...beside, [side.list]: capList(side, whole, c) };
|
|
489
|
+
}
|
|
490
|
+
// The sentence for each capped list: c, and the entries whose own list it cut, by name.
|
|
491
|
+
const capSaid = () => capping.flatMap((side) => {
|
|
492
|
+
const c = caps.get(side.list);
|
|
493
|
+
const kept = beside[side.list].length;
|
|
494
|
+
const inner = (e) => (Array.isArray(field(e, side.capped.list)) ? field(e, side.capped.list).length : 0);
|
|
495
|
+
const names = uncapped.get(side.list).slice(0, kept).filter((e) => inner(e) > c).map(side.capped.name);
|
|
496
|
+
return names.length ? [side.capped.said(c, names, place.full === true)] : [];
|
|
497
|
+
});
|
|
498
|
+
if (capping.length) {
|
|
499
|
+
const shown = lined({ ...beside, [cut.rows]: cutRows });
|
|
500
|
+
if (shown.length <= DATA_TEXT_BUDGET)
|
|
501
|
+
return { text: shown, said: say(rowsSaid, ...sidesSaid, ...capSaid(), cut.whole) };
|
|
502
|
+
}
|
|
503
|
+
const before = [];
|
|
504
|
+
for (const side of [...spent].reverse()) {
|
|
505
|
+
const v = beside[side.list];
|
|
506
|
+
if (!Array.isArray(v) || v.length === 0)
|
|
507
|
+
continue;
|
|
508
|
+
const total = Array.isArray(data[side.list]) ? data[side.list].length : v.length;
|
|
509
|
+
const at = (k) => lined({ ...beside, [side.list]: v.slice(0, k), [cut.rows]: cutRows }).length;
|
|
510
|
+
// Every entry of it does not fit (the text did not, as it stands): the most that do, or none.
|
|
511
|
+
let k = 0;
|
|
512
|
+
if (at(0) <= DATA_TEXT_BUDGET) {
|
|
513
|
+
let hi = v.length - 1;
|
|
514
|
+
while (k < hi) {
|
|
515
|
+
const mid = Math.ceil((k + hi) / 2);
|
|
516
|
+
if (at(mid) <= DATA_TEXT_BUDGET)
|
|
517
|
+
k = mid;
|
|
518
|
+
else
|
|
519
|
+
hi = mid - 1;
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
beside = { ...beside, [side.list]: v.slice(0, k) };
|
|
523
|
+
before.push(k === 0 ? `${side.list} keeps none of its ${total} entries` : `${side.list} keeps its first ${k} of ${total} entries`);
|
|
524
|
+
if (at(k) <= DATA_TEXT_BUDGET)
|
|
302
525
|
break;
|
|
303
|
-
out.add(i);
|
|
304
|
-
sum -= lens[i];
|
|
305
526
|
}
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
const
|
|
310
|
-
const
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
527
|
+
// The room the entries that left give back goes to the capped lists' c (as the structured cut's
|
|
528
|
+
// giveBack does), the first named first.
|
|
529
|
+
for (const side of capping) {
|
|
530
|
+
const whole = uncapped.get(side.list);
|
|
531
|
+
const kept = beside[side.list].length;
|
|
532
|
+
const c = mostCap(side, whole.slice(0, kept), beside);
|
|
533
|
+
if (c !== undefined && c > caps.get(side.list)) {
|
|
534
|
+
caps.set(side.list, c);
|
|
535
|
+
beside = { ...beside, [side.list]: capList(side, whole.slice(0, kept), c) };
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
if (before.length || capping.length) {
|
|
539
|
+
if (before.length)
|
|
540
|
+
sidesSaid = [...sidesSaid, `In it, ${listed(before)}, before any row of ${cut.rows} is left out.`];
|
|
541
|
+
sidesSaid = [...sidesSaid, ...capSaid()];
|
|
542
|
+
const shown = lined({ ...beside, [cut.rows]: cutRows });
|
|
543
|
+
if (shown.length <= DATA_TEXT_BUDGET)
|
|
544
|
+
return { text: shown, said: say(rowsSaid, ...sidesSaid, cut.whole) };
|
|
316
545
|
}
|
|
546
|
+
// Then rows leave the text, those the tool names first, then the rest from the end, until it
|
|
547
|
+
// fits (#2899: those it names last only once every other row is out).
|
|
548
|
+
const r = leaving(cutRows, beside, () => [rowsSaid, ...sidesSaid]);
|
|
549
|
+
if (r)
|
|
550
|
+
return r;
|
|
317
551
|
base = { ...beside, [cut.rows]: cutRows };
|
|
318
552
|
}
|
|
319
553
|
// Step 4: lists, then strings.
|
|
@@ -358,16 +592,25 @@ export const FULL_ARG_DESCRIPTION = "true: data in the structured result is the
|
|
|
358
592
|
export const FULL_ARG = z.boolean().optional().describe(FULL_ARG_DESCRIPTION);
|
|
359
593
|
// A level as its fields, and what it does inside each entry (#3132).
|
|
360
594
|
const levelOf = (l) => (Array.isArray(l) ? { keep: l } : l);
|
|
361
|
-
|
|
595
|
+
// The cap an entry is cut to at a level: `cap`, or the entry's floor where that is more (#3269's
|
|
596
|
+
// second review: EntryLevel.floor), unless `below` (leaveAt's floor, for the one entry kept).
|
|
597
|
+
const capFor = (lv, e, cap, below) => (lv.floor && !below ? Math.max(cap, lv.floor(e)) : cap);
|
|
598
|
+
const thinned = (e, level, cap = Number.POSITIVE_INFINITY, below) => {
|
|
362
599
|
if (!isPlainObject(e))
|
|
363
600
|
return e;
|
|
364
|
-
const
|
|
601
|
+
const lv = levelOf(level);
|
|
602
|
+
const { keep, capped, mark, pick } = lv;
|
|
603
|
+
const c = capFor(lv, e, cap, below);
|
|
365
604
|
const t = cutRow(e, keep, { keep, clip: Number.POSITIVE_INFINITY });
|
|
366
605
|
const inner = capped ? t[capped] : undefined;
|
|
367
|
-
if (capped && Array.isArray(inner) && inner.length >
|
|
368
|
-
|
|
369
|
-
|
|
606
|
+
if (capped && Array.isArray(inner) && inner.length > c) {
|
|
607
|
+
// cutRow keeps every entry of the list, in order, so inner[j] is the thinned e[capped][j].
|
|
608
|
+
const whole = e[capped];
|
|
609
|
+
t[capped] = pick && Array.isArray(whole) && whole.length === inner.length ? pick(whole, c).map((j) => inner[j]) : inner.slice(0, c);
|
|
610
|
+
}
|
|
611
|
+
return mark ? { ...t, ...mark(e, c) } : t;
|
|
370
612
|
};
|
|
613
|
+
const innerOf = (e, capped) => (capped !== undefined && Array.isArray(field(e, capped)) ? field(e, capped).length : 0);
|
|
371
614
|
// The longest list named `capped` inside any entry of a list (#3132): the cap a level starts below.
|
|
372
615
|
const longestInside = (v, capped) => Array.isArray(v) ? v.reduce((m, e) => Math.max(m, Array.isArray(field(e, capped)) ? field(e, capped).length : 0), 0) : 0;
|
|
373
616
|
// An entry JSON cannot write (undefined) is written null in a list.
|
|
@@ -381,27 +624,37 @@ const listOf = (data, at) => at.reduce((o, k) => field(o, k), data);
|
|
|
381
624
|
// (test/http-memory.test.mjs's) the structured cut allocated 108 MB, against 18 MB on 3.0.0 and 24
|
|
382
625
|
// MB now (V8's sampling heap profiler, collected objects included). What is measured and kept is
|
|
383
626
|
// the same: each entry thinned as before, its characters summed as before.
|
|
384
|
-
|
|
627
|
+
// #3269: with `outLevel`, the entries firstOut names are cut to it instead (DataCutStep.outFirst).
|
|
628
|
+
// #3269's second review: with `last`, the entries not named that it holds leave after those it does
|
|
629
|
+
// not (DataCutStep.leaveLast); with `below`, each entry is cut to `cap` under its floor.
|
|
630
|
+
function listAt(data, path, firstOut, level, newest, cap = Number.POSITIVE_INFINITY, levelAt, outLevel, last, below, ordered, shared) {
|
|
385
631
|
const at = path.split(".");
|
|
386
632
|
const v = listOf(data, at);
|
|
387
633
|
if (!Array.isArray(v) || v.length === 0)
|
|
388
634
|
return undefined;
|
|
389
635
|
const n = v.length;
|
|
390
|
-
const
|
|
636
|
+
const isNamed = (i) => firstOut !== undefined && firstOut.values.includes(field(v[i], firstOut.field));
|
|
637
|
+
const cutOne = (i) => (outLevel && isNamed(i) ? thinned(v[i], outLevel) : level ? thinned(v[i], level, cap, below) : v[i]);
|
|
638
|
+
const cache = shared ?? { made: [] };
|
|
639
|
+
const made = cache.made;
|
|
391
640
|
const entry = (i) => {
|
|
392
|
-
if (!level)
|
|
641
|
+
if (!level && !outLevel)
|
|
393
642
|
return v[i];
|
|
394
643
|
if (!(i in made))
|
|
395
|
-
made[i] =
|
|
644
|
+
made[i] = cutOne(i);
|
|
396
645
|
return made[i];
|
|
397
646
|
};
|
|
398
647
|
const thin = level ? levelOf(level).keep : undefined;
|
|
399
648
|
// A capped level's clause only where its cap cuts an entry's list (#3139's review: a package
|
|
400
|
-
// with one range is not "at most its first 1")
|
|
649
|
+
// with one range is not "at most its first 1"), each entry at its own floor or more (#3269's
|
|
650
|
+
// second review), and told whether some entry's floor is above the cap.
|
|
401
651
|
const lv = level ? levelOf(level) : undefined;
|
|
402
|
-
const bites = lv?.capped === undefined || v.some((e) => (
|
|
403
|
-
const
|
|
404
|
-
|
|
652
|
+
const bites = lv?.capped === undefined || v.some((e) => innerOf(e, lv.capped) > capFor(lv, e, cap, below));
|
|
653
|
+
const raised = lv?.capped !== undefined && lv.floor !== undefined && !below && v.some((e) => innerOf(e, lv.capped) > cap && lv.floor(e) > cap);
|
|
654
|
+
const thinSaid = lv && bites ? (below ? (lv.below ?? lv.said)?.(cap) : lv.said?.(cap, raised)) : undefined;
|
|
655
|
+
// The indices in the order they leave: from the end, or, for a series kept newest, from the start;
|
|
656
|
+
// those firstOut names first, value by value; then, with `last`, the others it does not hold,
|
|
657
|
+
// then those it holds.
|
|
405
658
|
const leavingOrder = (i) => (newest ? i : n - 1 - i);
|
|
406
659
|
const order = [];
|
|
407
660
|
for (const value of firstOut?.values ?? [])
|
|
@@ -410,20 +663,32 @@ function listAt(data, path, firstOut, level, newest, cap = Number.POSITIVE_INFIN
|
|
|
410
663
|
order.push(leavingOrder(j));
|
|
411
664
|
const named = order.length;
|
|
412
665
|
const queued = new Set(order);
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
666
|
+
const held = (i) => last !== undefined && last.has(v[i]);
|
|
667
|
+
for (const late of last ? [false, true] : [false])
|
|
668
|
+
for (let j = 0; j < n; j++)
|
|
669
|
+
if (!queued.has(leavingOrder(j)) && (!last || held(leavingOrder(j)) === late))
|
|
670
|
+
order.push(leavingOrder(j));
|
|
671
|
+
// #3269's second review: an order the cut chose (the rows kept by fit, cutData), named first.
|
|
672
|
+
if (ordered && ordered.length === n)
|
|
673
|
+
order.splice(0, n, ...ordered);
|
|
416
674
|
// The characters of the entries in the order they leave, summed: leaving[j] is the first j's.
|
|
417
675
|
let leaving;
|
|
676
|
+
const lensOf = () => {
|
|
677
|
+
if (cache.lens)
|
|
678
|
+
return cache.lens;
|
|
679
|
+
const lens = [];
|
|
680
|
+
for (let i = 0; i < n; i++)
|
|
681
|
+
lens.push(entryChars(i in made ? made[i] : cutOne(i)));
|
|
682
|
+
cache.lens = lens;
|
|
683
|
+
return lens;
|
|
684
|
+
};
|
|
418
685
|
const sums = () => {
|
|
419
686
|
if (leaving)
|
|
420
687
|
return leaving;
|
|
421
|
-
const
|
|
422
|
-
for (let i = 0; i < n; i++)
|
|
423
|
-
lens.push(entryChars(!level || i in made ? entry(i) : thinned(v[i], level, cap)));
|
|
688
|
+
const ls = lensOf();
|
|
424
689
|
leaving = [0];
|
|
425
690
|
for (const i of order)
|
|
426
|
-
leaving.push(leaving[leaving.length - 1] +
|
|
691
|
+
leaving.push(leaving[leaving.length - 1] + ls[i]);
|
|
427
692
|
return leaving;
|
|
428
693
|
};
|
|
429
694
|
const kept = (k) => Math.max(0, Math.min(n, k));
|
|
@@ -444,6 +709,21 @@ function listAt(data, path, firstOut, level, newest, cap = Number.POSITIVE_INFIN
|
|
|
444
709
|
...(levelAt !== undefined ? { level: levelAt } : {}),
|
|
445
710
|
cap,
|
|
446
711
|
over,
|
|
712
|
+
...(outLevel && named > 0 ? { outThin: levelOf(outLevel).keep } : {}),
|
|
713
|
+
isNamed,
|
|
714
|
+
...(last ? { leaveLast: last } : {}),
|
|
715
|
+
othersOut: (k) => {
|
|
716
|
+
const out = order.slice(0, n - kept(k)).filter((i) => !isNamed(i));
|
|
717
|
+
const own = (i) => (last ? field(v[i], last.id) : undefined);
|
|
718
|
+
const byPosition = out.some((i) => typeof own(i) !== "number" || !Number.isFinite(own(i)));
|
|
719
|
+
const id = (i) => (byPosition ? i + 1 : own(i));
|
|
720
|
+
const asc = (xs) => xs.sort((a, b) => id(a) - id(b));
|
|
721
|
+
return { held: asc(out.filter(held)), rest: asc(out.filter((i) => !held(i))), id, byPosition };
|
|
722
|
+
},
|
|
723
|
+
...(below ? { below } : {}),
|
|
724
|
+
order,
|
|
725
|
+
charsOf: (i) => lensOf()[i],
|
|
726
|
+
withOrder: (o) => listAt(data, path, firstOut, level, newest, cap, levelAt, outLevel, last, below, o, cache),
|
|
447
727
|
chars: (k) => {
|
|
448
728
|
const l = sums();
|
|
449
729
|
return l[n] - l[n - kept(k)] + Math.max(0, kept(k) - 1);
|
|
@@ -484,14 +764,73 @@ function givenBackSaid(list, k) {
|
|
|
484
764
|
const one = (r) => `${r.from === r.to ? `entry ${r.from}` : `entries ${r.from} to ${r.to}`} cut to ${keepSaid(r.g.keep)}${r.g.said ? `, ${r.g.said}` : ""}`;
|
|
485
765
|
return `; with the room left, ${n} of the entries kept take back detail (by position among those kept, in their order in data): ${runs.map(one).join("; ")}`;
|
|
486
766
|
}
|
|
767
|
+
// The most ids, or runs of them, the clause on the entries not named that left lists (#3269's
|
|
768
|
+
// second review): a client calls those again a few calls' worth at a time (workload_triage: at
|
|
769
|
+
// most 5 calls), and past this many it says how many more. #3269's third review: those LeaveLast
|
|
770
|
+
// holds and the others share it, the held ones first.
|
|
771
|
+
export const OTHERS_LEFT_LISTED = 100;
|
|
772
|
+
// Integers, ascending, as a sentence names them, runs of three or more as "a to b", at most `max`
|
|
773
|
+
// parts: the parts, how many integers they say (`n`) and leave out (`more`), and the last said.
|
|
774
|
+
const runsSaid = (xs, max = OTHERS_LEFT_LISTED) => {
|
|
775
|
+
const parts = [];
|
|
776
|
+
// xs[0..j) are said.
|
|
777
|
+
let j = 0;
|
|
778
|
+
while (j < xs.length && parts.length < max) {
|
|
779
|
+
let e = j;
|
|
780
|
+
while (e + 1 < xs.length && xs[e + 1] === xs[e] + 1)
|
|
781
|
+
e++;
|
|
782
|
+
if (e - j >= 2) {
|
|
783
|
+
parts.push(`${xs[j]} to ${xs[e]}`);
|
|
784
|
+
j = e + 1;
|
|
785
|
+
}
|
|
786
|
+
else {
|
|
787
|
+
parts.push(String(xs[j]));
|
|
788
|
+
j++;
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
return { said: listed(parts), parts: parts.length, n: j, more: xs.length - j, last: xs[j - 1] };
|
|
792
|
+
};
|
|
793
|
+
// The clause on the `g` entries not named that left, with LeaveLast: how many, the order they are
|
|
794
|
+
// kept in, and which (by its id), those LeaveLast holds first and apart (#3269's third review);
|
|
795
|
+
// and, where LeaveLast.again gives it, the sentence on checking them again: in that order, in calls
|
|
796
|
+
// of the sizes PageAt.againFit gives.
|
|
797
|
+
// With `counted` false the clause leaves out the count (the sentence has just said it). The order
|
|
798
|
+
// they were kept in is said where it shows (some entry kept after one that left, in input order).
|
|
799
|
+
function othersLeftSaid(list, k, g, fit, counted = true) {
|
|
800
|
+
const ll = list.leaveLast;
|
|
801
|
+
const { held, rest, id, byPosition } = list.othersOut(k);
|
|
802
|
+
const by = byPosition ? `position in ${list.path} (from 1)` : ll.id;
|
|
803
|
+
const h = runsSaid(held.map(id));
|
|
804
|
+
const r = runsSaid(rest.map(id), OTHERS_LEFT_LISTED - h.parts);
|
|
805
|
+
const at = (x) => x.n === 0 ? `not named here (the note names at most ${OTHERS_LEFT_LISTED} of them, or runs of them)` : `at ${by} ${x.said}${x.more > 0 ? `, and ${x.more} more past ${by} ${x.last}` : ""}`;
|
|
806
|
+
const which = held.length === 0
|
|
807
|
+
? `, ${g === 1 ? "without" : "none with"} ${ll.said}, ${at(r)}`
|
|
808
|
+
: rest.length === 0
|
|
809
|
+
? `, ${g === 1 ? "with" : "all with"} ${ll.said}, ${at(h)}`
|
|
810
|
+
: `: the ${held.length === 1 ? "one" : held.length} with ${ll.said} ${at(h)}; the other ${rest.length} ${at(r)}`;
|
|
811
|
+
// Where none of them is one LeaveLast holds and each left after every entry kept (the last
|
|
812
|
+
// ones), the order shows nothing to say (the synthetic's 201-purl probe: 43 rows kept with it
|
|
813
|
+
// unsaid, 42 with it said).
|
|
814
|
+
const kept = list.kept(k).filter((i) => !list.isNamed(i));
|
|
815
|
+
const last = held.length === 0 && (kept.length === 0 || rest.every((i) => i > kept[kept.length - 1]));
|
|
816
|
+
const said = `${counted ? `${g} ` : ""}${g === 1 ? ll.rest[0] : ll.rest[1]}${which}${last ? "" : ` (${ll.chosen})`}`;
|
|
817
|
+
if (!ll.again || byPosition)
|
|
818
|
+
return { said, again: "" };
|
|
819
|
+
// The entries the note names, in the order it names them.
|
|
820
|
+
const seq = [...held.slice(0, h.n), ...rest.slice(0, r.n)];
|
|
821
|
+
const { sizes, m, alone } = fit.againFit(list.path, seq);
|
|
822
|
+
const a = runsSaid(alone.map(id), Number.POSITIVE_INFINITY);
|
|
823
|
+
return { said, again: ll.again({ k: seq.length, sizes, m, alone: a.said, aloneCount: alone.length }) };
|
|
824
|
+
}
|
|
487
825
|
// What a cut keeps of each list it has cut so far, as truncated and as the DATA CUT sentences.
|
|
488
|
-
function cutSaid(cut, kept, wholeChars, data, over,
|
|
826
|
+
function cutSaid(cut, kept, wholeChars, data, over, fit) {
|
|
489
827
|
const isThinned = ({ list, k }) => list.thin !== undefined && k > 0;
|
|
490
828
|
const cutOnes = kept.filter((x) => x.k < x.list.n || isThinned(x));
|
|
491
829
|
if (cutOnes.length === 0)
|
|
492
830
|
return { said: "" };
|
|
493
831
|
const truncated = Object.fromEntries(cutOnes.map((x) => [x.list.path, { total: x.list.n, returned: x.k, ...(isThinned(x) ? { thinned: true } : {}) }]));
|
|
494
832
|
const parts = [];
|
|
833
|
+
const again = [];
|
|
495
834
|
const byKept = new Map();
|
|
496
835
|
for (const x of cutOnes) {
|
|
497
836
|
const { list, k } = x;
|
|
@@ -499,9 +838,29 @@ function cutSaid(cut, kept, wholeChars, data, over, wholeFit) {
|
|
|
499
838
|
const first = Math.min(out, list.named);
|
|
500
839
|
const end = list.newest ? "oldest" : "last";
|
|
501
840
|
const which = list.firstOut ? `whose ${list.firstOut.field} is ${orListed(list.firstOut.values)} (${list.firstOut.why})` : "";
|
|
502
|
-
|
|
841
|
+
// #3269's second review: with LeaveLast, the entries not named that left are named by its id,
|
|
842
|
+
// so a client can call them alone again, and the order they leave in is said. #3269's third
|
|
843
|
+
// review: and how many of them, in the order named, a call of them alone keeps every one of.
|
|
844
|
+
const others = list.leaveLast && out > first ? othersLeftSaid(list, k, out - first, fit, first > 0) : undefined;
|
|
845
|
+
const rest = others?.said ?? "";
|
|
846
|
+
if (others?.again)
|
|
847
|
+
again.push(others.again);
|
|
848
|
+
const leftOut = out === 0
|
|
849
|
+
? ""
|
|
850
|
+
: rest
|
|
851
|
+
? first === 0
|
|
852
|
+
? `, the ${out} left out being ${rest}`
|
|
853
|
+
: `, the ${out} left out being the ${first} ${which} and ${rest}`
|
|
854
|
+
: first === 0
|
|
855
|
+
? `, the ${end} ${out} left out`
|
|
856
|
+
: first === out
|
|
857
|
+
? `, the ${out} left out being entries ${which}`
|
|
858
|
+
: `, the ${out} left out being the ${first} ${which} and the ${end} ${out - first} of the rest`;
|
|
503
859
|
if (isThinned(x)) {
|
|
504
|
-
|
|
860
|
+
// #3269: the entries outFirst cut apart, those of them kept.
|
|
861
|
+
const apart = list.outThin ? list.kept(k).filter((i) => list.isNamed(i)).length : 0;
|
|
862
|
+
const apartSaid = apart > 0 && list.firstOut ? `, except the ${apart} kept whose ${list.firstOut.field} is ${orListed(list.firstOut.values)}, each cut to ${keepSaid(list.outThin)}` : "";
|
|
863
|
+
parts.push(`${list.path} keeps ${k === list.n ? `all ${k}` : `${k} of its ${list.n}`} entries, each cut to ${keepSaid(list.thin)}${list.thinSaid ? `, ${list.thinSaid}` : ""}${apartSaid}${leftOut}${givenBackSaid(list, k)}`);
|
|
505
864
|
continue;
|
|
506
865
|
}
|
|
507
866
|
if (!list.firstOut || first === 0 || k === 0) {
|
|
@@ -529,9 +888,10 @@ function cutSaid(cut, kept, wholeChars, data, over, wholeFit) {
|
|
|
529
888
|
return p.k === 0 ? `${paths} keep none of their entries` : `${paths} keep their ${which} ${p.k} entries each`;
|
|
530
889
|
});
|
|
531
890
|
const keeps = Object.fromEntries(cutOnes.filter(isThinned).map((x) => [x.list.path, x.list.thin]));
|
|
532
|
-
const page = cut.page?.(truncated, data, { keeps,
|
|
891
|
+
const page = cut.page?.(truncated, data, { keeps, ...fit }) ?? "";
|
|
533
892
|
const said = [
|
|
534
893
|
`DATA CUT: with data whole, this structured result is ${wholeChars} characters of JSON, more than the ${STRUCTURED_BUDGET} it holds unless the call says full: true, so its data is cut: ${cuts.join("; ")}.`,
|
|
894
|
+
...again,
|
|
535
895
|
"truncated gives the total and the number returned of each list cut, and thinned where its entries are cut to some of their fields.",
|
|
536
896
|
page,
|
|
537
897
|
over ? `Cut so, the structured result is still past ${STRUCTURED_BUDGET} characters: data holds no other list to cut.` : "",
|
|
@@ -552,6 +912,8 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
552
912
|
const steps = [...cut.steps, { lists: Object.keys(data).filter((k) => !named.has(k)) }];
|
|
553
913
|
// Each list cut so far, in the order cut, at the level it was thinned to, how many entries it
|
|
554
914
|
// keeps, and the step that cut it.
|
|
915
|
+
// `gone` (#3269): the list's entries outFirst names left before the others were cut further, and
|
|
916
|
+
// may come back once the cut is settled.
|
|
555
917
|
const kept = new Map();
|
|
556
918
|
const apply = (also = []) => [...kept.values(), ...also].reduce((d, { list, k }) => withList(d, list.at, list.keep(k)), data);
|
|
557
919
|
// PageAt.wholeFit: of the list at `path`, the most first entries, each whole, that the result
|
|
@@ -577,13 +939,244 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
577
939
|
wholeFits.set(path, k);
|
|
578
940
|
return k;
|
|
579
941
|
};
|
|
942
|
+
// PageAt.firstFit (#3269's review): of the list at `path`, which a step thins, the most first
|
|
943
|
+
// entries n such that, for every n' up to n, a call answering only the first n' of them keeps
|
|
944
|
+
// every one of them, as this cut would cut that call: it does where
|
|
945
|
+
// - every entry fits thinned (thin[0]), those leaveOutFirst names cut to outFirst's level; or
|
|
946
|
+
// - every entry fits at the floor (thin[floor.at] at its least, each entry at its own
|
|
947
|
+
// EntryLevel.floor or more: #3269's second review), those named so cut, and either none is
|
|
948
|
+
// named or the entries not named, thinned, are past the budget on their own (so the call
|
|
949
|
+
// cannot keep them uncapped by sending the named ones out, and keeps them at the floor
|
|
950
|
+
// instead).
|
|
951
|
+
// A step without a floor is held to its leaveAt level and cap instead, so a page of that size
|
|
952
|
+
// comes back with that level's detail, as #3132's review asked. Each size is counted as this
|
|
953
|
+
// call's: the entries as they are (the first n of a call are these, at the same index), data
|
|
954
|
+
// beside them with this list and the lists of the steps before it empty (a cut empties those
|
|
955
|
+
// first, and a call of fewer purls sends them all), and this call's envelope and note, with the
|
|
956
|
+
// longest DATA CUT sentence such a call can carry while it keeps every entry. A call of fewer
|
|
957
|
+
// purls has a note no longer than this one's (its counts are no larger, the affected components
|
|
958
|
+
// it names are the first of these), but for its TEXT CUT sentences, which can be cut no further
|
|
959
|
+
// than this call's text but can take another form: FIRST_FIT_SLACK holds the difference.
|
|
960
|
+
const FIRST_FIT_SLACK = 600;
|
|
961
|
+
const measures = new Map();
|
|
962
|
+
const measureOf = (path) => {
|
|
963
|
+
const known = measures.get(path);
|
|
964
|
+
if (known !== undefined)
|
|
965
|
+
return known;
|
|
966
|
+
const s = steps.findIndex((st) => st.lists.includes(path));
|
|
967
|
+
const v = listOf(data, path.split("."));
|
|
968
|
+
const m = s >= 0 && steps[s].thin?.length && Array.isArray(v) && v.length > 0 ? measureFor(steps[s], s, path, v) : null;
|
|
969
|
+
measures.set(path, m);
|
|
970
|
+
return m;
|
|
971
|
+
};
|
|
972
|
+
const measureFor = (step, s, path, v) => {
|
|
973
|
+
const thin = step.thin;
|
|
974
|
+
const first = step.leaveOutFirst;
|
|
975
|
+
const out = first ? step.outFirst : undefined;
|
|
976
|
+
const floor = out?.floor;
|
|
977
|
+
const lean = out?.level;
|
|
978
|
+
const n = v.length;
|
|
979
|
+
const isNamed = (e) => first !== undefined && first.values.includes(field(e, first.field));
|
|
980
|
+
const bare = [path, ...steps.slice(0, s).flatMap((st) => st.lists)].reduce((d, p) => (Array.isArray(listOf(d, p.split("."))) ? withList(d, p.split("."), []) : d), data);
|
|
981
|
+
const base = JSON.stringify(bare).length;
|
|
982
|
+
// The level and cap every entry is held to where they are not all kept thinned.
|
|
983
|
+
const atFloor = floor ? floor.at : step.leaveAt ? step.leaveAt.level : thin.length - 1;
|
|
984
|
+
const floorCap = floor ? (levelOf(thin[atFloor]).least ?? 0) : step.leaveAt ? step.leaveAt.cap : Number.POSITIVE_INFINITY;
|
|
985
|
+
// The envelope, its note and the longest DATA CUT sentence of a call that keeps every entry,
|
|
986
|
+
// at each level it can stop at (a capped level's clause said at its least, where it bites).
|
|
987
|
+
const noPage = { steps: cut.steps };
|
|
988
|
+
const stops = [0, ...(floor && out ? Array.from({ length: floor.at - out.before + 1 }, (_, j) => out.before + j) : [atFloor])];
|
|
989
|
+
const env = Math.max(...stops.map((j) => {
|
|
990
|
+
const lv = levelOf(thin[j]);
|
|
991
|
+
const list = listAt(data, path, first, thin[j], step.keepNewest, lv.capped ? (lv.least ?? 0) : Number.POSITIVE_INFINITY, j, lean, step.leaveLast);
|
|
992
|
+
if (!list)
|
|
993
|
+
return 0;
|
|
994
|
+
const { said } = cutSaid(noPage, [{ list, k: n }], wholeChars, data, false, { wholeFit, firstFit: () => 0, againFit: () => ({ sizes: [], m: 0, alone: [] }) });
|
|
995
|
+
return JSON.stringify(structured(null, { [path]: { total: n, returned: n, thinned: true } }, said)).length - "null".length;
|
|
996
|
+
})) +
|
|
997
|
+
base +
|
|
998
|
+
FIRST_FIT_SLACK;
|
|
999
|
+
const chars = (e, level, cap = Number.POSITIVE_INFINITY) => entryChars(thinned(e, level, cap));
|
|
1000
|
+
const once = (f) => {
|
|
1001
|
+
const got = new Map();
|
|
1002
|
+
return (i) => {
|
|
1003
|
+
let c = got.get(i);
|
|
1004
|
+
if (c === undefined)
|
|
1005
|
+
got.set(i, (c = f(i)));
|
|
1006
|
+
return c;
|
|
1007
|
+
};
|
|
1008
|
+
};
|
|
1009
|
+
const named = (i) => isNamed(v[i]);
|
|
1010
|
+
return {
|
|
1011
|
+
v,
|
|
1012
|
+
step,
|
|
1013
|
+
env,
|
|
1014
|
+
named,
|
|
1015
|
+
thinChars: once((i) => chars(v[i], named(i) && lean !== undefined ? lean : thin[0])),
|
|
1016
|
+
floorChars: once((i) => (named(i) && lean !== undefined ? chars(v[i], lean) : chars(v[i], thin[atFloor], floorCap))),
|
|
1017
|
+
};
|
|
1018
|
+
};
|
|
1019
|
+
const firstFits = new Map();
|
|
1020
|
+
const firstFit = (path) => {
|
|
1021
|
+
const known = firstFits.get(path);
|
|
1022
|
+
if (known !== undefined)
|
|
1023
|
+
return known;
|
|
1024
|
+
const ms = measureOf(path);
|
|
1025
|
+
const fit = ms ? firstFitOf(ms) : 0;
|
|
1026
|
+
firstFits.set(path, fit);
|
|
1027
|
+
return fit;
|
|
1028
|
+
};
|
|
1029
|
+
const firstFitOf = ({ v, step, env, named: isNamed, thinChars, floorChars }) => {
|
|
1030
|
+
const out = step.leaveOutFirst ? step.outFirst : undefined;
|
|
1031
|
+
const floor = out?.floor;
|
|
1032
|
+
const n = v.length;
|
|
1033
|
+
let thinnedAll = 0;
|
|
1034
|
+
let atFloorAll = 0;
|
|
1035
|
+
let others = 0;
|
|
1036
|
+
let othersAlone = 0;
|
|
1037
|
+
let named = 0;
|
|
1038
|
+
for (let i = 0; i < n; i++) {
|
|
1039
|
+
const nm = isNamed(i);
|
|
1040
|
+
if (nm)
|
|
1041
|
+
named++;
|
|
1042
|
+
thinnedAll += thinChars(i);
|
|
1043
|
+
atFloorAll += floorChars(i);
|
|
1044
|
+
if (!nm) {
|
|
1045
|
+
others++;
|
|
1046
|
+
othersAlone += thinChars(i);
|
|
1047
|
+
}
|
|
1048
|
+
const commas = i;
|
|
1049
|
+
const keepsThinned = env + thinnedAll + commas <= STRUCTURED_BUDGET;
|
|
1050
|
+
const keepsAtFloor = env + atFloorAll + commas <= STRUCTURED_BUDGET;
|
|
1051
|
+
// The entries not named, thinned, without the rest of the result: a lower bound.
|
|
1052
|
+
const othersPast = othersAlone + Math.max(0, others - 1) + 2 > STRUCTURED_BUDGET;
|
|
1053
|
+
if (!(keepsThinned || (keepsAtFloor && (named === 0 || !out || (floor !== undefined && othersPast)))))
|
|
1054
|
+
return i;
|
|
1055
|
+
}
|
|
1056
|
+
return n;
|
|
1057
|
+
};
|
|
1058
|
+
// PageAt.againFit (#3269's third review): of the entries `seq` names (none of them one
|
|
1059
|
+
// leaveOutFirst names, so a call of them alone keeps every one where they fit at their floor, as
|
|
1060
|
+
// firstFit counts), those that do not fit at it even alone leave `seq` (`alone`). Of the rest,
|
|
1061
|
+
// each of the first LeaveLast.againCalls calls takes the most of those left, in order, that fit
|
|
1062
|
+
// at the floor beside the result's envelope (this call's, its note with LeaveLast.noteOver's
|
|
1063
|
+
// characters for a call of that many, and a DATA CUT sentence keeping them all); of the entries
|
|
1064
|
+
// after those, m is the most such that every m of them in a row fit so. So each call of those
|
|
1065
|
+
// sizes keeps every one, and so does a call of fewer of them in a row. Counted once per `seq`
|
|
1066
|
+
// (the cut sizes the same note many times).
|
|
1067
|
+
const againFits = new Map();
|
|
1068
|
+
const againFit = (path, seq) => {
|
|
1069
|
+
const key = `${path}:${seq.join(",")}`;
|
|
1070
|
+
const known = againFits.get(key);
|
|
1071
|
+
if (known)
|
|
1072
|
+
return known;
|
|
1073
|
+
const got = againFitOf(path, seq);
|
|
1074
|
+
againFits.set(key, got);
|
|
1075
|
+
return got;
|
|
1076
|
+
};
|
|
1077
|
+
const againFitOf = (path, seq) => {
|
|
1078
|
+
const ms = measureOf(path);
|
|
1079
|
+
if (!ms || seq.length === 0)
|
|
1080
|
+
return { sizes: [], m: 0, alone: [] };
|
|
1081
|
+
const ll = ms.step.leaveLast;
|
|
1082
|
+
const over = ll?.noteOver?.(seq.map((i) => ms.v[i]), ms.v) ?? (() => 0);
|
|
1083
|
+
const room = (m) => STRUCTURED_BUDGET - ms.env - over(m);
|
|
1084
|
+
const one = room(1);
|
|
1085
|
+
const alone = seq.filter((i) => ms.floorChars(i) > one);
|
|
1086
|
+
const rest = alone.length ? seq.filter((i) => ms.floorChars(i) <= one) : seq;
|
|
1087
|
+
const calls = ll?.againCalls ?? 0;
|
|
1088
|
+
// sums[j]: the characters of rest's first j entries; rest[a..b) fits as one call.
|
|
1089
|
+
const sums = [0];
|
|
1090
|
+
for (const i of rest)
|
|
1091
|
+
sums.push(sums[sums.length - 1] + ms.floorChars(i));
|
|
1092
|
+
const fitsAt = (a, b) => sums[b] - sums[a] + (b - a - 1) <= room(b - a);
|
|
1093
|
+
const sizes = [];
|
|
1094
|
+
let at = 0;
|
|
1095
|
+
while (sizes.length < calls && at < rest.length) {
|
|
1096
|
+
let b = at + 1;
|
|
1097
|
+
while (b < rest.length && fitsAt(at, b + 1))
|
|
1098
|
+
b++;
|
|
1099
|
+
sizes.push(b - at);
|
|
1100
|
+
at = b;
|
|
1101
|
+
}
|
|
1102
|
+
// Every m of rest[at..] in a row fits (fewer, at its end, are within the last m).
|
|
1103
|
+
const fits = (m) => {
|
|
1104
|
+
for (let a = at; a + m <= rest.length; a++)
|
|
1105
|
+
if (!fitsAt(a, a + m))
|
|
1106
|
+
return false;
|
|
1107
|
+
return true;
|
|
1108
|
+
};
|
|
1109
|
+
let m = rest.length > at ? 1 : 0;
|
|
1110
|
+
let hi = rest.length - at;
|
|
1111
|
+
while (m < hi) {
|
|
1112
|
+
const mid = Math.ceil((m + hi) / 2);
|
|
1113
|
+
if (fits(mid))
|
|
1114
|
+
m = mid;
|
|
1115
|
+
else
|
|
1116
|
+
hi = mid - 1;
|
|
1117
|
+
}
|
|
1118
|
+
return { sizes, m, alone };
|
|
1119
|
+
};
|
|
1120
|
+
const fit = { wholeFit, firstFit, againFit };
|
|
580
1121
|
// The result's size with this step's lists at k entries each: the rest of the result (data
|
|
581
1122
|
// null), data with this step's lists empty (`base`), and their entries.
|
|
582
1123
|
const size = (lists, k, base) => {
|
|
583
1124
|
const all = [...kept.values(), ...lists.map((list) => ({ list, k: Math.min(k, list.n) }))];
|
|
584
|
-
const { truncated, said } = cutSaid(cut, all, wholeChars, data, false,
|
|
1125
|
+
const { truncated, said } = cutSaid(cut, all, wholeChars, data, false, fit);
|
|
585
1126
|
return JSON.stringify(structured(null, truncated, said)).length - "null".length + base + lists.reduce((t, l) => t + l.chars(k), 0);
|
|
586
1127
|
};
|
|
1128
|
+
// #3269's second review: whether an entry of a step's lists that leaveOutFirst does not name holds
|
|
1129
|
+
// any entry of the list leaveAt's level caps (DataCutStep.leaveAt's floor).
|
|
1130
|
+
const holdsInner = (step) => {
|
|
1131
|
+
const inner = step.thin && step.leaveAt ? levelOf(step.thin[step.leaveAt.level]).capped : undefined;
|
|
1132
|
+
if (inner === undefined)
|
|
1133
|
+
return false;
|
|
1134
|
+
const first = step.leaveOutFirst;
|
|
1135
|
+
const named = (e) => first !== undefined && first.values.includes(field(e, first.field));
|
|
1136
|
+
return step.lists.some((p) => {
|
|
1137
|
+
const v = listOf(data, p.split("."));
|
|
1138
|
+
return Array.isArray(v) && v.some((e) => !named(e) && innerOf(e, inner) > 0);
|
|
1139
|
+
});
|
|
1140
|
+
};
|
|
1141
|
+
// #3269's second review: with leaveAt's floor and LeaveLast, the entries kept are chosen in the
|
|
1142
|
+
// reverse of the order they leave in (LeaveLast's first, then the rest, each from the start),
|
|
1143
|
+
// each one kept that fits beside those chosen before it, and the rest leave first: the list so
|
|
1144
|
+
// reordered, and how many it keeps. In the plain order a row too large to fit beside the rows kept
|
|
1145
|
+
// sent every row before it in that order out with it, and left their room empty: a row of 200
|
|
1146
|
+
// KEV-listed matches after 150 purls of an image (40 affected) kept 1 row of 151, in 7,572
|
|
1147
|
+
// characters. The choice is made on the room the result leaves with none kept, then checked as
|
|
1148
|
+
// the result is (its note names the entries left out, by index), the last chosen leaving while it
|
|
1149
|
+
// does not fit, and topped up, each entry not chosen that fits now, in the same order.
|
|
1150
|
+
const keptByFit = (l, base) => {
|
|
1151
|
+
if (size([l], 0, base) > STRUCTURED_BUDGET)
|
|
1152
|
+
return undefined;
|
|
1153
|
+
const named = l.order.filter((i) => l.isNamed(i));
|
|
1154
|
+
const pri = [...l.order].reverse().filter((i) => !l.isNamed(i));
|
|
1155
|
+
const orderOf = (chosen) => [...named, ...l.order.filter((i) => !l.isNamed(i) && !chosen.has(i)), ...l.order.filter((i) => chosen.has(i))];
|
|
1156
|
+
const chosen = new Set();
|
|
1157
|
+
let room = STRUCTURED_BUDGET - size([l], 0, base);
|
|
1158
|
+
for (const i of pri) {
|
|
1159
|
+
const c = l.charsOf(i) + (chosen.size > 0 ? 1 : 0);
|
|
1160
|
+
if (c > room)
|
|
1161
|
+
continue;
|
|
1162
|
+
chosen.add(i);
|
|
1163
|
+
room -= c;
|
|
1164
|
+
}
|
|
1165
|
+
let cur = l.withOrder(orderOf(chosen));
|
|
1166
|
+
let k = chosen.size;
|
|
1167
|
+
while (k > 0 && size([cur], k, base) > STRUCTURED_BUDGET)
|
|
1168
|
+
k--;
|
|
1169
|
+
for (let tries = 0; tries < 64; tries++) {
|
|
1170
|
+
const kept = new Set(cur.kept(k));
|
|
1171
|
+
const left = STRUCTURED_BUDGET - size([cur], k, base);
|
|
1172
|
+
const next = pri.find((i) => !kept.has(i) && cur.charsOf(i) + 1 <= left && size([cur.withOrder(orderOf(new Set([...kept, i])))], k + 1, base) <= STRUCTURED_BUDGET);
|
|
1173
|
+
if (next === undefined)
|
|
1174
|
+
break;
|
|
1175
|
+
cur = cur.withOrder(orderOf(new Set([...kept, next])));
|
|
1176
|
+
k++;
|
|
1177
|
+
}
|
|
1178
|
+
return { list: cur, k };
|
|
1179
|
+
};
|
|
587
1180
|
let fits = false;
|
|
588
1181
|
for (const [s, step] of steps.entries()) {
|
|
589
1182
|
// Every entry whole, then each thinning level in turn, every entry kept; at the leanest level
|
|
@@ -591,13 +1184,105 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
591
1184
|
const levels = [undefined, ...(step.thin ?? [])];
|
|
592
1185
|
let lists = [];
|
|
593
1186
|
let k = 0;
|
|
594
|
-
|
|
1187
|
+
// #3269: once the entries outFirst names have left (gone), each later level cuts them to its
|
|
1188
|
+
// level, so that they can come back so cut, and keeps at most the others (they leave first).
|
|
1189
|
+
const out = step.leaveOutFirst ? step.outFirst : undefined;
|
|
1190
|
+
let gone = false;
|
|
1191
|
+
const at = (level, cap, levelAt, outLevel = gone ? out?.level : undefined, below) => step.lists
|
|
1192
|
+
.map((p) => listAt(data, p, step.leaveOutFirst, level, step.keepNewest, cap, levelAt, outLevel, step.leaveLast, below))
|
|
1193
|
+
.filter((l) => l !== undefined && !kept.has(l.path));
|
|
1194
|
+
// #3269's second review: with leaveAt's floor, the last level tried with every entry kept is
|
|
1195
|
+
// leaveAt's while an entry not named holds inner entries (DataCutStep.leaveAt): no later level
|
|
1196
|
+
// keeps any of them, so the entries leave there, whole, instead.
|
|
1197
|
+
const end = step.leaveAt?.floor && step.thin && holdsInner(step) ? step.leaveAt.level + 1 : levels.length - 1;
|
|
595
1198
|
for (const [i, thin] of levels.entries()) {
|
|
1199
|
+
if (i > end)
|
|
1200
|
+
break;
|
|
1201
|
+
// #3269: before any entry is cut to thin[out.before], the entries leaveOutFirst names are cut
|
|
1202
|
+
// to out.level, the others kept at the level before; then they leave, as many as must.
|
|
1203
|
+
if (out && !gone && i === out.before + 1) {
|
|
1204
|
+
const apart = at(levels[i - 1], undefined, i - 2, out.level);
|
|
1205
|
+
if (apart.some((l) => l.named > 0)) {
|
|
1206
|
+
const base = JSON.stringify(apply(apart.map((list) => ({ list, k: 0 })))).length;
|
|
1207
|
+
const rest = Math.max(...apart.map((l) => l.n - l.named));
|
|
1208
|
+
const all = Math.max(...apart.map((l) => l.n));
|
|
1209
|
+
// The most entries each list keeps, `lo` at least, cut as `ls` is.
|
|
1210
|
+
const most = (ls, lo) => {
|
|
1211
|
+
let kk = lo;
|
|
1212
|
+
let hi = all;
|
|
1213
|
+
while (kk < hi) {
|
|
1214
|
+
const mid = Math.ceil((kk + hi) / 2);
|
|
1215
|
+
if (size(ls, mid, base) <= STRUCTURED_BUDGET)
|
|
1216
|
+
kk = mid;
|
|
1217
|
+
else
|
|
1218
|
+
hi = mid - 1;
|
|
1219
|
+
}
|
|
1220
|
+
return kk;
|
|
1221
|
+
};
|
|
1222
|
+
if (size(apart, rest, base) <= STRUCTURED_BUDGET) {
|
|
1223
|
+
k = most(apart, rest);
|
|
1224
|
+
lists = apart;
|
|
1225
|
+
fits = true;
|
|
1226
|
+
break;
|
|
1227
|
+
}
|
|
1228
|
+
// #3269's review: the others are capped anyway, so the named entries stay while the
|
|
1229
|
+
// others fit at the floor (OutFirst.floor). #3269's second review: each capped level
|
|
1230
|
+
// holds each entry's own floor (EntryLevel.floor), so the cap starts at its least.
|
|
1231
|
+
const floor = out.floor;
|
|
1232
|
+
if (floor) {
|
|
1233
|
+
const last = floor.at + 1;
|
|
1234
|
+
// With kk entries kept, the richest capped level from thin[before] to thin[floor.at]
|
|
1235
|
+
// that fits at the floor or more, at the most inner entries that fit; none where none.
|
|
1236
|
+
const capped = (kk) => {
|
|
1237
|
+
for (let j = i; j <= last; j++) {
|
|
1238
|
+
const lv = levelOf(levels[j]);
|
|
1239
|
+
if (lv.capped === undefined)
|
|
1240
|
+
continue;
|
|
1241
|
+
const ls = (c) => at(levels[j], c, j - 1, out.level);
|
|
1242
|
+
let m = lv.least ?? 0;
|
|
1243
|
+
if (size(ls(m), kk, base) > STRUCTURED_BUDGET)
|
|
1244
|
+
continue;
|
|
1245
|
+
let hi = Math.max(m, ...step.lists.map((p) => longestInside(listOf(data, p.split(".")), lv.capped)));
|
|
1246
|
+
while (m < hi) {
|
|
1247
|
+
const mid = Math.ceil((m + hi) / 2);
|
|
1248
|
+
if (size(ls(mid), kk, base) <= STRUCTURED_BUDGET)
|
|
1249
|
+
m = mid;
|
|
1250
|
+
else
|
|
1251
|
+
hi = mid - 1;
|
|
1252
|
+
}
|
|
1253
|
+
return ls(m);
|
|
1254
|
+
}
|
|
1255
|
+
return undefined;
|
|
1256
|
+
};
|
|
1257
|
+
let kk = all;
|
|
1258
|
+
let got = capped(all);
|
|
1259
|
+
if (!got) {
|
|
1260
|
+
// Even the floor does not fit with every named entry: they leave, as many as must
|
|
1261
|
+
// for it to, and the others then take the most that fits.
|
|
1262
|
+
const lv = levelOf(levels[last]);
|
|
1263
|
+
const atLeast = at(levels[last], lv.least ?? 0, last - 1, out.level);
|
|
1264
|
+
if (size(atLeast, rest, base) <= STRUCTURED_BUDGET) {
|
|
1265
|
+
kk = most(atLeast, rest);
|
|
1266
|
+
got = capped(kk);
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
if (got) {
|
|
1270
|
+
lists = got;
|
|
1271
|
+
k = kk;
|
|
1272
|
+
if (kk < all)
|
|
1273
|
+
gone = true;
|
|
1274
|
+
fits = true;
|
|
1275
|
+
break;
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
gone = true;
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
596
1281
|
lists = at(thin, undefined, i - 1);
|
|
597
1282
|
if (lists.length === 0)
|
|
598
1283
|
break;
|
|
599
1284
|
const base = JSON.stringify(apply(lists.map((list) => ({ list, k: 0 })))).length;
|
|
600
|
-
const top = Math.max(...lists.map((l) => l.n));
|
|
1285
|
+
const top = Math.max(...lists.map((l) => (gone ? l.n - l.named : l.n)));
|
|
601
1286
|
// #3132: a level that caps a list inside each entry, every entry kept: the most entries of
|
|
602
1287
|
// that inner list, the same number in each, that fit; a level where even none fits is passed.
|
|
603
1288
|
// #3132's review: a level with `least` is taken at that cap or more, or passed.
|
|
@@ -620,12 +1305,12 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
620
1305
|
fits = true;
|
|
621
1306
|
break;
|
|
622
1307
|
}
|
|
623
|
-
if (i <
|
|
1308
|
+
if (i < end)
|
|
624
1309
|
continue;
|
|
625
1310
|
// The last level, and even its least cap does not fit: the entries go on at that cap.
|
|
626
1311
|
lists = at(thin, least, i - 1);
|
|
627
1312
|
}
|
|
628
|
-
if (i <
|
|
1313
|
+
if (i < end) {
|
|
629
1314
|
if (size(lists, top, base) <= STRUCTURED_BUDGET) {
|
|
630
1315
|
k = top;
|
|
631
1316
|
fits = true;
|
|
@@ -652,7 +1337,13 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
652
1337
|
}
|
|
653
1338
|
break;
|
|
654
1339
|
}
|
|
655
|
-
|
|
1340
|
+
const chosen = step.leaveAt?.floor && step.leaveLast && lists.length === 1 ? keptByFit(lists[0], base) : undefined;
|
|
1341
|
+
if (chosen) {
|
|
1342
|
+
lists = [chosen.list];
|
|
1343
|
+
k = chosen.k;
|
|
1344
|
+
fits = true;
|
|
1345
|
+
}
|
|
1346
|
+
else if (size(lists, 0, base) <= STRUCTURED_BUDGET) {
|
|
656
1347
|
// The most entries each list of the step keeps with the result inside the budget.
|
|
657
1348
|
let hi = top;
|
|
658
1349
|
while (k < hi) {
|
|
@@ -664,14 +1355,47 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
664
1355
|
}
|
|
665
1356
|
fits = true;
|
|
666
1357
|
}
|
|
1358
|
+
// #3269's second review: not even one entry fits at its floor (one extreme row, its KEV-listed
|
|
1359
|
+
// matches alone past the budget). The one the leaving order keeps last stays, its capped list
|
|
1360
|
+
// at the most that fit below its floor, at least one; where not even one fits, at the first
|
|
1361
|
+
// later level that fits; and only where none does, no entry.
|
|
1362
|
+
if (fits && k === 0 && top > 0 && step.leaveAt?.floor && step.thin) {
|
|
1363
|
+
const { level } = step.leaveAt;
|
|
1364
|
+
const inner = levelOf(step.thin[level]).capped;
|
|
1365
|
+
let got;
|
|
1366
|
+
if (inner !== undefined) {
|
|
1367
|
+
const ls = (c) => at(step.thin?.[level], c, level, undefined, true);
|
|
1368
|
+
if (size(ls(1), 1, base) <= STRUCTURED_BUDGET) {
|
|
1369
|
+
let c = 1;
|
|
1370
|
+
let hi = Math.max(1, ...step.lists.map((p) => longestInside(listOf(data, p.split(".")), inner))) - 1;
|
|
1371
|
+
while (c < hi) {
|
|
1372
|
+
const mid = Math.ceil((c + hi) / 2);
|
|
1373
|
+
if (size(ls(mid), 1, base) <= STRUCTURED_BUDGET)
|
|
1374
|
+
c = mid;
|
|
1375
|
+
else
|
|
1376
|
+
hi = mid - 1;
|
|
1377
|
+
}
|
|
1378
|
+
got = ls(c);
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
for (let j = level + 1; !got && j < step.thin.length; j++) {
|
|
1382
|
+
const ls = at(step.thin[j], undefined, j);
|
|
1383
|
+
if (size(ls, 1, base) <= STRUCTURED_BUDGET)
|
|
1384
|
+
got = ls;
|
|
1385
|
+
}
|
|
1386
|
+
if (got) {
|
|
1387
|
+
lists = got;
|
|
1388
|
+
k = 1;
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
667
1391
|
}
|
|
668
1392
|
for (const list of lists)
|
|
669
|
-
kept.set(list.path, { list, k: Math.min(k, list.n), step: s });
|
|
1393
|
+
kept.set(list.path, { list, k: Math.min(k, list.n), step: s, ...(gone ? { gone: true } : {}) });
|
|
670
1394
|
if (fits)
|
|
671
1395
|
break;
|
|
672
1396
|
}
|
|
673
1397
|
const exact = () => {
|
|
674
|
-
const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, false,
|
|
1398
|
+
const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, false, fit);
|
|
675
1399
|
return JSON.stringify(structured(apply(), truncated, said)).length;
|
|
676
1400
|
};
|
|
677
1401
|
// #3139: every step cut and the result still past the budget: a keepAll step's entries leave now,
|
|
@@ -759,7 +1483,8 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
759
1483
|
for (const e of [...kept.values()].filter((x) => x.step === s)) {
|
|
760
1484
|
const { list } = e;
|
|
761
1485
|
const own = list.level;
|
|
762
|
-
|
|
1486
|
+
// #3269's second review: the one entry kept below its floor takes back nothing (more did not fit).
|
|
1487
|
+
if (own === undefined || list.thin === undefined || list.below)
|
|
763
1488
|
continue;
|
|
764
1489
|
const whole = listOf(data, list.at);
|
|
765
1490
|
if (!Array.isArray(whole))
|
|
@@ -777,13 +1502,16 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
777
1502
|
return false;
|
|
778
1503
|
};
|
|
779
1504
|
for (const i of list.kept(e.k)) {
|
|
1505
|
+
// #3269: an entry outFirst cut apart takes back nothing (the others come first).
|
|
1506
|
+
if (list.outThin && list.isNamed(i))
|
|
1507
|
+
continue;
|
|
780
1508
|
// An entry its own level already keeps as the richest would is passed, not an end.
|
|
781
1509
|
if (JSON.stringify(thinned(whole[i], thin[0])) === JSON.stringify(thinned(whole[i], thin[own], list.cap)))
|
|
782
1510
|
continue;
|
|
783
1511
|
let took = false;
|
|
784
1512
|
for (let j = 0; j <= own && !took; j++) {
|
|
785
1513
|
const l = levelOf(thin[j]);
|
|
786
|
-
const inner =
|
|
1514
|
+
const inner = innerOf(whole[i], l.capped);
|
|
787
1515
|
if (l.capped === undefined) {
|
|
788
1516
|
if (j < own)
|
|
789
1517
|
took = tryAt(i, j, Number.POSITIVE_INFINITY);
|
|
@@ -791,11 +1519,13 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
791
1519
|
}
|
|
792
1520
|
if (j !== leanestCapped)
|
|
793
1521
|
continue;
|
|
794
|
-
|
|
1522
|
+
// #3269's second review: the cap the entry is cut to, its floor where that is more.
|
|
1523
|
+
const now = capFor(l, whole[i], list.cap);
|
|
1524
|
+
if (j < own || now < inner)
|
|
795
1525
|
took = tryAt(i, j, Number.POSITIVE_INFINITY);
|
|
796
1526
|
if (took)
|
|
797
1527
|
break;
|
|
798
|
-
let c = j === own ?
|
|
1528
|
+
let c = j === own ? now + 1 : capFor(l, whole[i], l.least ?? 0);
|
|
799
1529
|
let hi = inner - 1;
|
|
800
1530
|
if (c > hi || !tryAt(i, j, c))
|
|
801
1531
|
continue;
|
|
@@ -812,7 +1542,27 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
812
1542
|
}
|
|
813
1543
|
}
|
|
814
1544
|
}
|
|
815
|
-
|
|
1545
|
+
// #3269: last, the entries outFirst sent out come back with the room left, cut to its level, the
|
|
1546
|
+
// last to leave first (they leave first of all, so they come back only where every other entry
|
|
1547
|
+
// stayed), as many as fit.
|
|
1548
|
+
if (fits) {
|
|
1549
|
+
for (const e of kept.values()) {
|
|
1550
|
+
if (!e.gone || e.k < e.list.n - e.list.named)
|
|
1551
|
+
continue;
|
|
1552
|
+
let hi = e.list.n;
|
|
1553
|
+
let k = e.k;
|
|
1554
|
+
while (k < hi) {
|
|
1555
|
+
const mid = Math.ceil((k + hi) / 2);
|
|
1556
|
+
e.k = mid;
|
|
1557
|
+
if (exact() <= STRUCTURED_BUDGET)
|
|
1558
|
+
k = mid;
|
|
1559
|
+
else
|
|
1560
|
+
hi = mid - 1;
|
|
1561
|
+
}
|
|
1562
|
+
e.k = k;
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, !fits, fit);
|
|
816
1566
|
if (truncated === undefined)
|
|
817
1567
|
return undefined;
|
|
818
1568
|
const d = apply();
|