echelongraph-mcp 3.0.2 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -32,9 +32,20 @@ import * as z from "zod";
32
32
  // in "…", every list in it to its first `cap` entries); then, for a tool that names the
33
33
  // lists beside its rows (`sides`: check_affected's excluded sample, check_sbom's
34
34
  // not_sent_purls), the same level with those lists cut as it says; then each next level with
35
- // them cut; and when no level fits, rows are left out of the text, those the tool names
36
- // first, then from the end. A list beside the rows is cut before any row field the first
37
- // level keeps, and before any row leaves the text;
35
+ // them cut; and when no level fits, the lists beside the rows that the tool names
36
+ // beforeRows (#3272: cve_remediation's fixed_branches, patches and fix_references, which its
37
+ // structured cut thins and leaves in the same order) are cut as each says, one after another,
38
+ // then (3.1.0's phase A review, `capped`: fixed_branches) each entry's own list is cut to its
39
+ // first c entries, the most that fit, every entry kept and marked as the structured cut
40
+ // marks it, and then they lose entries from the end, the last named first, the room left
41
+ // going back to c; then rows are left out of the
42
+ // text, those the tool names first, then from the end, those it names last (leaveOutLast,
43
+ // #2899: check_affected's KEV-listed and ransomware matches) only after every other, the note
44
+ // naming each of those left out. A list the tool names beside its rows is cut before any row
45
+ // leaves the text, and, unless it is named beforeRows, before any row field the first level
46
+ // keeps (a list it does not name is cut only in step 4).
47
+ // With outFirstBefore (#3269: check_sbom), the rows the tool names are cut to the leanest
48
+ // level and leave first, as many as must, before any other row is cut to the level it names;
38
49
  // 4. else every list keeps its first n entries, n halving, and then strings are cut too.
39
50
  // Whenever the text is not the JSON whole, the note ends with one sentence, starting TEXT CUT,
40
51
  // that says what the first text block leaves out, where structuredContent.data carries all of it,
@@ -66,7 +77,17 @@ import * as z from "zod";
66
77
  // each entry on its own terms (an EntryLevel, #3132: check_sbom's rows keep at most the same
67
78
  // number of matches, the most that fit, so only the rows with more lose any), and a step named
68
79
  // keepAll (#3139: cve_remediation's vendor_remediations) keeps every entry at its leanest level
69
- // while the later steps are cut, its entries leaving only once they all are;
80
+ // while the later steps are cut, its entries leaving only once they all are; and a step named
81
+ // with outFirst (#3269: check_sbom's results) sends the entries leaveOutFirst names out, cut
82
+ // lean, before the others are cut past a given level, and has them back with the room left,
83
+ // but (#3269's review, outFirst.floor) keeps them while the others, capped anyway, fit at a
84
+ // floor; and a tool that pages names the page size at which a call of its first entries keeps
85
+ // them all (PageAt.firstFit), counted as this cut would cut that call; #3269's second review:
86
+ // with EntryLevel.floor and leaveAt's floor no entry's list goes below its floor, the entries
87
+ // leave whole instead, chosen by fit, those DataCutStep.leaveLast holds kept first, and the
88
+ // note names by id those of them that left (#3269's third review: those it holds first and
89
+ // apart, and, where the tool gives the sentence, the sizes of calls of them alone that keep
90
+ // every one, PageAt.againFit);
70
91
  // 2. then every other list at the top level of data, the same way. A list inside an object of
71
92
  // data (get_cve's remediation summary, with fix_status, #2841) is cut only where a DataCut
72
93
  // names it, and none names get_cve's. Once a step fits, the room it leaves goes back to the
@@ -242,6 +263,7 @@ export const FIRST_TEXT_BLOCK = {
242
263
  export const FIRST_TEXT_BLOCK_BESIDE_A_CUT = {
243
264
  opens: (chars) => `TEXT CUT: the API's answer is ${chars} characters of JSON, more than the ${DATA_TEXT_BUDGET} the first text block holds, so that block is cut, and structuredContent.data carries the answer whole with full: true (DATA CUT, below).`,
244
265
  leftOut: (n, which) => `the other ${n} are left out of it${which}, and structuredContent.data holds them with full: true`,
266
+ full: true,
245
267
  };
246
268
  /** The first text block for `data` within DATA_TEXT_BUDGET, and the TEXT CUT sentence when it is cut. */
247
269
  export function dataText(data, cut, place = FIRST_TEXT_BLOCK) {
@@ -261,59 +283,271 @@ export function dataText(data, cut, place = FIRST_TEXT_BLOCK) {
261
283
  if (cut && isPlainObject(data) && Array.isArray(rows) && rows.length > 0 && cut.levels.length > 0) {
262
284
  const n = rows.length;
263
285
  // The lists beside the rows, cut once; tried with the first level after it fails with them
264
- // whole, and with every later level.
265
- const sides = cut.sides?.length ? cutSides(data, cut.sides) : { fields: {}, said: [] };
286
+ // whole, and with every later level. (Those named beforeRows, #3272, are cut only below.)
287
+ const early = (cut.sides ?? []).filter((side) => !side.beforeRows);
288
+ const sides = early.length ? cutSides(data, early) : { fields: {}, said: [] };
266
289
  const sided = sides.said.length > 0;
267
290
  const attempts = cut.levels.flatMap((level, i) => (i === 0 && sided ? [{ level, sided: false }, { level, sided: true }] : [{ level, sided }]));
291
+ // The order rows leave the text in: those the tool names first (by value, in its order, each
292
+ // from the end), then the rest from the end.
293
+ const firstOut = cut.leaveOutFirst;
294
+ const isNamed = (i) => firstOut !== undefined && firstOut.values.includes(String(field(rows[i], firstOut.field)));
295
+ const lastOut = cut.leaveOutLast;
296
+ const isLast = (i) => lastOut !== undefined && (lastOut.fields.some((f) => field(rows[i], f) === true) || lastOut.has?.(rows[i]) === true);
297
+ // What a row named last has, as the note says it: "kev_listed or ransomware true" (#2899), or
298
+ // the tool's own words (#3284: "a KEV-listed or ransomware match").
299
+ const lastHas = lastOut ? (lastOut.said ?? `${orListed(lastOut.fields)} true`) : "";
300
+ const lastWhose = lastOut ? (lastOut.said ? `with ${lastOut.said}` : `whose ${orListed(lastOut.fields)} is true`) : "";
301
+ const lastWithout = lastOut ? (lastOut.said ? `without ${lastOut.said}` : `whose ${orListed(lastOut.fields)} is not true`) : "";
302
+ const order = [];
303
+ for (const value of firstOut?.values ?? []) {
304
+ for (let i = n - 1; i >= 0; i--)
305
+ if (field(rows[i], firstOut?.field ?? "") === value)
306
+ order.push(i);
307
+ }
308
+ const named = order.length;
309
+ const queued = new Set(order);
310
+ for (let i = n - 1; i >= 0; i--)
311
+ if (!queued.has(i) && !isLast(i))
312
+ order.push(i);
313
+ for (let i = n - 1; i >= 0; i--)
314
+ if (!queued.has(i) && isLast(i))
315
+ order.push(i);
316
+ // Rows leave `cutRows` in that order, the first `upTo` of it at most, until the text fits; the
317
+ // answer so cut, its sentences after those `said` gives for the number of named rows that left,
318
+ // or undefined where it does not fit.
319
+ const leaving = (cutRows, beside, said, upTo = n) => {
320
+ const lens = cutRows.map((r) => 4 + JSON.stringify(r).length);
321
+ const emptyLen = lined({ ...beside, [cut.rows]: [] }).length;
322
+ const textLen = (kept, sum) => (kept === 0 ? emptyLen : emptyLen + 2 + sum + 2 * kept);
323
+ const out = new Set();
324
+ let sum = lens.reduce((t, x) => t + x, 0);
325
+ for (const i of order.slice(0, upTo)) {
326
+ if (textLen(n - out.size, sum) <= DATA_TEXT_BUDGET)
327
+ break;
328
+ out.add(i);
329
+ sum -= lens[i];
330
+ }
331
+ const kept = cutRows.filter((_, i) => !out.has(i));
332
+ const shown = lined({ ...beside, [cut.rows]: kept });
333
+ if (kept.length === 0 || shown.length > DATA_TEXT_BUDGET)
334
+ return undefined;
335
+ const outFirst = [...out].filter(isNamed).length;
336
+ // #2899: the rows left out are the last in order unless a row named last stayed behind them.
337
+ const inOrder = [...out].every((i) => i >= n - out.size);
338
+ // #3284: the same of the rows left out that the tool does not name first: the last of the
339
+ // rest unless a row named last stayed behind them.
340
+ const restOut = [...out].filter((i) => !isNamed(i));
341
+ const restIdx = [...Array(n).keys()].filter((i) => !isNamed(i));
342
+ const restInOrder = restOut.every((i) => restIdx.indexOf(i) >= restIdx.length - restOut.length);
343
+ const butLast = lastOut ? ` but for any ${lastWhose}${lastOut.why ? ` (${lastOut.why})` : ""}, which leave it last` : "";
344
+ // #3284: the rest, where a row named last stayed behind them: the last of those without one,
345
+ // or, where rows named last left too, each without one and then the last of those with one
346
+ // (said short: the note counts against the structured result's budget).
347
+ const lastOutN = [...out].filter(isLast).length;
348
+ const restOf = (m) => restInOrder || !lastOut
349
+ ? `, and the last ${m} of the rest`
350
+ : lastOutN === 0
351
+ ? `, and the last ${m} of the rest ${lastWithout}`
352
+ : `, and ${m} of the rest: each ${lastWithout}, then the last ${lastOutN} ${lastWhose}`;
353
+ const which = outFirst > 0
354
+ ? `: the ${outFirst} whose ${firstOut?.field} is ${(firstOut?.values ?? []).join(" or ")} (${firstOut?.why})${out.size > outFirst ? restOf(out.size - outFirst) : ""}`
355
+ : inOrder || !lastOut
356
+ ? `, the last ${out.size} in order`
357
+ : !lastOut.said
358
+ ? `, the last ${out.size} in order${butLast}`
359
+ : lastOutN === 0
360
+ ? `, the last ${out.size} ${lastWithout}`
361
+ : `: each ${lastWithout}, then the last ${lastOutN} ${lastWhose}`;
362
+ const elsewhere = place.leftOut ? place.leftOut(out.size, which) : `the other ${out.size} are in structuredContent.data only${which}`;
363
+ const left = out.size > 0 ? `Only ${kept.length} of the ${n} rows of ${cut.rows} are in it; ${elsewhere}.` : "";
364
+ // #2899: each row named last that the text still leaves out, by name.
365
+ const lastLeft = [...out].filter(isLast).sort((a, b) => a - b);
366
+ const named_ = () => {
367
+ if (!lastOut?.runs)
368
+ return listed(lastLeft.map((i) => String(field(rows[i], lastOut?.name ?? "") ?? `${cut.rows}[${i}]`)));
369
+ // #3284: by their numbers, in runs, at most OTHERS_LEFT_LISTED parts.
370
+ const ids = lastLeft.map((i) => field(rows[i], lastOut.name)).map((v, j) => (typeof v === "number" ? v : lastLeft[j])).sort((a, b) => a - b);
371
+ const r = runsSaid(ids);
372
+ return `at ${lastOut.name} ${r.said}${r.more > 0 ? `, and ${r.more} more past ${lastOut.name} ${r.last}` : ""}`;
373
+ };
374
+ const lastSaid = lastOut && lastLeft.length ? `Of the rows left out of it, ${lastLeft.length} ${lastLeft.length === 1 ? "has" : "have"} ${lastHas}: ${named_()}.` : "";
375
+ return { text: shown, said: say(...said(outFirst, out), left, lastSaid, cut.whole, out.size > 0 ? cut.page?.(kept.length, n, data) : undefined) };
376
+ };
268
377
  let cutRows = [];
269
378
  let beside = data;
270
379
  for (const attempt of attempts) {
271
380
  const { level } = attempt;
381
+ beside = attempt.sided ? { ...data, ...sides.fields } : data;
382
+ sidesSaid = attempt.sided ? sides.said : [];
383
+ // #3269: before any row is cut to the level outFirstBefore names, the rows leaveOutFirst
384
+ // names are cut to the leanest level, the others kept at the level before, and then leave,
385
+ // as many as must. Where even that does not fit, the cut goes on as it did through 3.0.2:
386
+ // every row at the next level, then rows leaving, those named first.
387
+ const li = cut.levels.indexOf(level);
388
+ if (cut.outFirstBefore !== undefined && li === cut.outFirstBefore && li > 0 && named > 0 && attempt === attempts.find((a) => a.level === level)) {
389
+ const prev = cut.levels[li - 1];
390
+ const lean = cut.levels[cut.levels.length - 1];
391
+ const s = { left: new Set(), clipped: 0, capped: 0 };
392
+ const sn = { left: new Set(), clipped: 0, capped: 0 };
393
+ // #3269's review: the named rows whose strings the lean level clips, or whose lists it
394
+ // caps, by index, so the sentence says so of the ones kept (round 1 said only the fields
395
+ // they leave out, and a clean row's 150-character purl came back clipped unsaid).
396
+ const clippedAt = new Set();
397
+ const cappedAt = new Set();
398
+ const apart = rows.map((r, i) => {
399
+ if (!isPlainObject(r))
400
+ return cutValue(r, prev.clip, prev.cap ?? Number.POSITIVE_INFINITY, s);
401
+ if (!isNamed(i))
402
+ return cutRow(r, prev.keep, prev, s);
403
+ const [clipped, capped] = [sn.clipped, sn.capped];
404
+ const lean_ = cutRow(r, lean.keep, lean, sn);
405
+ if (sn.clipped > clipped)
406
+ clippedAt.add(i);
407
+ if (sn.capped > capped)
408
+ cappedAt.add(i);
409
+ return lean_;
410
+ });
411
+ const extra = [...sn.left].filter((f) => !s.left.has(f));
412
+ const apartSaid = (left, out) => {
413
+ const parts = [];
414
+ if (extra.length)
415
+ parts.push(`leave out ${listed(extra)} too`);
416
+ if ([...clippedAt].some((i) => !out.has(i)))
417
+ parts.push(`every string in each of them longer than ${lean.clip} characters keeps its first ${lean.clip}, ending in "…"`);
418
+ if (lean.cap !== undefined && [...cappedAt].some((i) => !out.has(i)))
419
+ parts.push(`every list in each of them longer than ${lean.cap} entries keeps its first ${lean.cap}`);
420
+ return named - left > 0 && parts.length ? `The ${named - left} rows of ${cut.rows} in it whose ${firstOut?.field} is ${(firstOut?.values ?? []).join(" or ")} ${parts.join("; ")}.` : "";
421
+ };
422
+ const prevSaid = levelSaid(cut.rows, n, prev, s);
423
+ const shown = lined({ ...beside, [cut.rows]: apart });
424
+ if (shown.length <= DATA_TEXT_BUDGET)
425
+ return { text: shown, said: say(prevSaid, apartSaid(0, new Set()), ...sidesSaid, cut.whole) };
426
+ // Only the named rows leave here.
427
+ const r = leaving(apart, beside, (left, out) => [prevSaid, apartSaid(left, out), ...sidesSaid], named);
428
+ if (r)
429
+ return r;
430
+ }
272
431
  const s = { left: new Set(), clipped: 0, capped: 0 };
273
432
  const cap = level.cap ?? Number.POSITIVE_INFINITY;
274
433
  cutRows = rows.map((r) => (isPlainObject(r) ? cutRow(r, level.keep, level, s) : cutValue(r, level.clip, cap, s)));
275
434
  rowsSaid = levelSaid(cut.rows, n, level, s);
276
- beside = attempt.sided ? { ...data, ...sides.fields } : data;
277
- sidesSaid = attempt.sided ? sides.said : [];
278
435
  const shown = lined({ ...beside, [cut.rows]: cutRows });
279
436
  if (shown.length <= DATA_TEXT_BUDGET)
280
437
  return { text: shown, said: say(rowsSaid, ...sidesSaid, cut.whole) };
281
438
  }
282
- // No level fits: rows leave the text, those the tool names first (by value, in its order,
283
- // each from the end), then the rest from the end, until it fits.
284
- const lens = cutRows.map((r) => 4 + JSON.stringify(r).length);
285
- const emptyLen = lined({ ...beside, [cut.rows]: [] }).length;
286
- const textLen = (kept, sum) => (kept === 0 ? emptyLen : emptyLen + 2 + sum + 2 * kept);
287
- const firstOut = cut.leaveOutFirst;
288
- const order = [];
289
- for (const value of firstOut?.values ?? []) {
290
- for (let i = n - 1; i >= 0; i--)
291
- if (field(rows[i], firstOut?.field ?? "") === value)
292
- order.push(i);
439
+ // No level fits. #3272: the lists the tool names beforeRows, whole until now, are cut as each
440
+ // says, one after another in the order named, until the text fits with every row at the
441
+ // leanest level; then they lose entries, from the end, the last named first, each emptied
442
+ // before the next loses any (a DataCut's keepAll steps, in the same order); the note says
443
+ // what each keeps.
444
+ const spent = (cut.sides ?? []).filter((side) => side.beforeRows);
445
+ for (const side of spent) {
446
+ const one = cutSides(data, [side]);
447
+ if (!one.said.length)
448
+ continue;
449
+ beside = { ...beside, ...one.fields };
450
+ sidesSaid = [...sidesSaid, ...one.said];
451
+ const shown = lined({ ...beside, [cut.rows]: cutRows });
452
+ if (shown.length <= DATA_TEXT_BUDGET)
453
+ return { text: shown, said: say(rowsSaid, ...sidesSaid, cut.whole) };
293
454
  }
294
- const queued = new Set(order);
295
- for (let i = n - 1; i >= 0; i--)
296
- if (!queued.has(i))
297
- order.push(i);
298
- const out = new Set();
299
- let sum = lens.reduce((t, x) => t + x, 0);
300
- for (const i of order) {
301
- if (textLen(n - out.size, sum) <= DATA_TEXT_BUDGET)
455
+ // 3.1.0's phase A review: the lists named with `capped` have each entry's own list cut to its
456
+ // first c entries, the most that fit, before any entry of a list leaves (#3272's text emptied
457
+ // fixed_branches once one package of some 400 ranges did not fit alone, while 3.0.2's every-list
458
+ // cap kept 32 packages; data caps each package's ranges instead, and so does the text now).
459
+ const capping = spent.filter((side) => side.capped && Array.isArray(beside[side.list]) && beside[side.list].length > 0);
460
+ const caps = new Map();
461
+ const capList = (side, v, c) => v.map((e) => {
462
+ const inner = isPlainObject(e) ? e[side.capped.list] : undefined;
463
+ return isPlainObject(e) && Array.isArray(inner) && inner.length > c ? { ...e, [side.capped.list]: inner.slice(0, c), ...side.capped.mark(e, c) } : e;
464
+ });
465
+ const longest = (v, list) => v.reduce((m, e) => Math.max(m, Array.isArray(field(e, list)) ? field(e, list).length : 0), 0);
466
+ // The most c (from `least` up to the longest list less one) at which the text fits, or undefined.
467
+ const mostCap = (side, whole, with_) => {
468
+ const at = (c) => lined({ ...with_, [side.list]: capList(side, whole, c), [cut.rows]: cutRows }).length <= DATA_TEXT_BUDGET;
469
+ let lo = side.capped.least;
470
+ let hi = longest(whole, side.capped.list) - 1;
471
+ if (hi < lo || !at(lo))
472
+ return undefined;
473
+ while (lo < hi) {
474
+ const mid = Math.ceil((lo + hi) / 2);
475
+ if (at(mid))
476
+ lo = mid;
477
+ else
478
+ hi = mid - 1;
479
+ }
480
+ return lo;
481
+ };
482
+ const uncapped = new Map();
483
+ for (const side of capping) {
484
+ const whole = beside[side.list];
485
+ uncapped.set(side.list, whole);
486
+ const c = mostCap(side, whole, beside) ?? side.capped.least;
487
+ caps.set(side.list, c);
488
+ beside = { ...beside, [side.list]: capList(side, whole, c) };
489
+ }
490
+ // The sentence for each capped list: c, and the entries whose own list it cut, by name.
491
+ const capSaid = () => capping.flatMap((side) => {
492
+ const c = caps.get(side.list);
493
+ const kept = beside[side.list].length;
494
+ const inner = (e) => (Array.isArray(field(e, side.capped.list)) ? field(e, side.capped.list).length : 0);
495
+ const names = uncapped.get(side.list).slice(0, kept).filter((e) => inner(e) > c).map(side.capped.name);
496
+ return names.length ? [side.capped.said(c, names, place.full === true)] : [];
497
+ });
498
+ if (capping.length) {
499
+ const shown = lined({ ...beside, [cut.rows]: cutRows });
500
+ if (shown.length <= DATA_TEXT_BUDGET)
501
+ return { text: shown, said: say(rowsSaid, ...sidesSaid, ...capSaid(), cut.whole) };
502
+ }
503
+ const before = [];
504
+ for (const side of [...spent].reverse()) {
505
+ const v = beside[side.list];
506
+ if (!Array.isArray(v) || v.length === 0)
507
+ continue;
508
+ const total = Array.isArray(data[side.list]) ? data[side.list].length : v.length;
509
+ const at = (k) => lined({ ...beside, [side.list]: v.slice(0, k), [cut.rows]: cutRows }).length;
510
+ // Every entry of it does not fit (the text did not, as it stands): the most that do, or none.
511
+ let k = 0;
512
+ if (at(0) <= DATA_TEXT_BUDGET) {
513
+ let hi = v.length - 1;
514
+ while (k < hi) {
515
+ const mid = Math.ceil((k + hi) / 2);
516
+ if (at(mid) <= DATA_TEXT_BUDGET)
517
+ k = mid;
518
+ else
519
+ hi = mid - 1;
520
+ }
521
+ }
522
+ beside = { ...beside, [side.list]: v.slice(0, k) };
523
+ before.push(k === 0 ? `${side.list} keeps none of its ${total} entries` : `${side.list} keeps its first ${k} of ${total} entries`);
524
+ if (at(k) <= DATA_TEXT_BUDGET)
302
525
  break;
303
- out.add(i);
304
- sum -= lens[i];
305
526
  }
306
- const kept = cutRows.filter((_, i) => !out.has(i));
307
- const shown = lined({ ...beside, [cut.rows]: kept });
308
- if (kept.length > 0 && shown.length <= DATA_TEXT_BUDGET) {
309
- const outFirst = firstOut ? [...out].filter((i) => firstOut.values.includes(String(field(rows[i], firstOut.field)))).length : 0;
310
- const which = outFirst > 0
311
- ? `: the ${outFirst} whose ${firstOut?.field} is ${(firstOut?.values ?? []).join(" or ")} (${firstOut?.why})${out.size > outFirst ? `, and the last ${out.size - outFirst} of the rest` : ""}`
312
- : `, the last ${out.size} in order`;
313
- const elsewhere = place.leftOut ? place.leftOut(out.size, which) : `the other ${out.size} are in structuredContent.data only${which}`;
314
- const left = `Only ${kept.length} of the ${n} rows of ${cut.rows} are in it; ${elsewhere}.`;
315
- return { text: shown, said: say(rowsSaid, ...sidesSaid, left, cut.whole, cut.page?.(kept.length, n, data)) };
527
+ // The room the entries that left give back goes to the capped lists' c (as the structured cut's
528
+ // giveBack does), the first named first.
529
+ for (const side of capping) {
530
+ const whole = uncapped.get(side.list);
531
+ const kept = beside[side.list].length;
532
+ const c = mostCap(side, whole.slice(0, kept), beside);
533
+ if (c !== undefined && c > caps.get(side.list)) {
534
+ caps.set(side.list, c);
535
+ beside = { ...beside, [side.list]: capList(side, whole.slice(0, kept), c) };
536
+ }
537
+ }
538
+ if (before.length || capping.length) {
539
+ if (before.length)
540
+ sidesSaid = [...sidesSaid, `In it, ${listed(before)}, before any row of ${cut.rows} is left out.`];
541
+ sidesSaid = [...sidesSaid, ...capSaid()];
542
+ const shown = lined({ ...beside, [cut.rows]: cutRows });
543
+ if (shown.length <= DATA_TEXT_BUDGET)
544
+ return { text: shown, said: say(rowsSaid, ...sidesSaid, cut.whole) };
316
545
  }
546
+ // Then rows leave the text, those the tool names first, then the rest from the end, until it
547
+ // fits (#2899: those it names last only once every other row is out).
548
+ const r = leaving(cutRows, beside, () => [rowsSaid, ...sidesSaid]);
549
+ if (r)
550
+ return r;
317
551
  base = { ...beside, [cut.rows]: cutRows };
318
552
  }
319
553
  // Step 4: lists, then strings.
@@ -358,16 +592,25 @@ export const FULL_ARG_DESCRIPTION = "true: data in the structured result is the
358
592
  export const FULL_ARG = z.boolean().optional().describe(FULL_ARG_DESCRIPTION);
359
593
  // A level as its fields, and what it does inside each entry (#3132).
360
594
  const levelOf = (l) => (Array.isArray(l) ? { keep: l } : l);
361
- const thinned = (e, level, cap = Number.POSITIVE_INFINITY) => {
595
+ // The cap an entry is cut to at a level: `cap`, or the entry's floor where that is more (#3269's
596
+ // second review: EntryLevel.floor), unless `below` (leaveAt's floor, for the one entry kept).
597
+ const capFor = (lv, e, cap, below) => (lv.floor && !below ? Math.max(cap, lv.floor(e)) : cap);
598
+ const thinned = (e, level, cap = Number.POSITIVE_INFINITY, below) => {
362
599
  if (!isPlainObject(e))
363
600
  return e;
364
- const { keep, capped, mark } = levelOf(level);
601
+ const lv = levelOf(level);
602
+ const { keep, capped, mark, pick } = lv;
603
+ const c = capFor(lv, e, cap, below);
365
604
  const t = cutRow(e, keep, { keep, clip: Number.POSITIVE_INFINITY });
366
605
  const inner = capped ? t[capped] : undefined;
367
- if (capped && Array.isArray(inner) && inner.length > cap)
368
- t[capped] = inner.slice(0, cap);
369
- return mark ? { ...t, ...mark(e, cap) } : t;
606
+ if (capped && Array.isArray(inner) && inner.length > c) {
607
+ // cutRow keeps every entry of the list, in order, so inner[j] is the thinned e[capped][j].
608
+ const whole = e[capped];
609
+ t[capped] = pick && Array.isArray(whole) && whole.length === inner.length ? pick(whole, c).map((j) => inner[j]) : inner.slice(0, c);
610
+ }
611
+ return mark ? { ...t, ...mark(e, c) } : t;
370
612
  };
613
+ const innerOf = (e, capped) => (capped !== undefined && Array.isArray(field(e, capped)) ? field(e, capped).length : 0);
371
614
  // The longest list named `capped` inside any entry of a list (#3132): the cap a level starts below.
372
615
  const longestInside = (v, capped) => Array.isArray(v) ? v.reduce((m, e) => Math.max(m, Array.isArray(field(e, capped)) ? field(e, capped).length : 0), 0) : 0;
373
616
  // An entry JSON cannot write (undefined) is written null in a list.
@@ -381,27 +624,37 @@ const listOf = (data, at) => at.reduce((o, k) => field(o, k), data);
381
624
  // (test/http-memory.test.mjs's) the structured cut allocated 108 MB, against 18 MB on 3.0.0 and 24
382
625
  // MB now (V8's sampling heap profiler, collected objects included). What is measured and kept is
383
626
  // the same: each entry thinned as before, its characters summed as before.
384
- function listAt(data, path, firstOut, level, newest, cap = Number.POSITIVE_INFINITY, levelAt) {
627
+ // #3269: with `outLevel`, the entries firstOut names are cut to it instead (DataCutStep.outFirst).
628
+ // #3269's second review: with `last`, the entries not named that it holds leave after those it does
629
+ // not (DataCutStep.leaveLast); with `below`, each entry is cut to `cap` under its floor.
630
+ function listAt(data, path, firstOut, level, newest, cap = Number.POSITIVE_INFINITY, levelAt, outLevel, last, below, ordered, shared) {
385
631
  const at = path.split(".");
386
632
  const v = listOf(data, at);
387
633
  if (!Array.isArray(v) || v.length === 0)
388
634
  return undefined;
389
635
  const n = v.length;
390
- const made = [];
636
+ const isNamed = (i) => firstOut !== undefined && firstOut.values.includes(field(v[i], firstOut.field));
637
+ const cutOne = (i) => (outLevel && isNamed(i) ? thinned(v[i], outLevel) : level ? thinned(v[i], level, cap, below) : v[i]);
638
+ const cache = shared ?? { made: [] };
639
+ const made = cache.made;
391
640
  const entry = (i) => {
392
- if (!level)
641
+ if (!level && !outLevel)
393
642
  return v[i];
394
643
  if (!(i in made))
395
- made[i] = thinned(v[i], level, cap);
644
+ made[i] = cutOne(i);
396
645
  return made[i];
397
646
  };
398
647
  const thin = level ? levelOf(level).keep : undefined;
399
648
  // A capped level's clause only where its cap cuts an entry's list (#3139's review: a package
400
- // with one range is not "at most its first 1").
649
+ // with one range is not "at most its first 1"), each entry at its own floor or more (#3269's
650
+ // second review), and told whether some entry's floor is above the cap.
401
651
  const lv = level ? levelOf(level) : undefined;
402
- const bites = lv?.capped === undefined || v.some((e) => (Array.isArray(field(e, lv.capped)) ? field(e, lv.capped).length : 0) > cap);
403
- const thinSaid = lv && bites ? lv.said?.(cap) : undefined;
404
- // The indices in the order they leave: from the end, or, for a series kept newest, from the start.
652
+ const bites = lv?.capped === undefined || v.some((e) => innerOf(e, lv.capped) > capFor(lv, e, cap, below));
653
+ const raised = lv?.capped !== undefined && lv.floor !== undefined && !below && v.some((e) => innerOf(e, lv.capped) > cap && lv.floor(e) > cap);
654
+ const thinSaid = lv && bites ? (below ? (lv.below ?? lv.said)?.(cap) : lv.said?.(cap, raised)) : undefined;
655
+ // The indices in the order they leave: from the end, or, for a series kept newest, from the start;
656
+ // those firstOut names first, value by value; then, with `last`, the others it does not hold,
657
+ // then those it holds.
405
658
  const leavingOrder = (i) => (newest ? i : n - 1 - i);
406
659
  const order = [];
407
660
  for (const value of firstOut?.values ?? [])
@@ -410,20 +663,32 @@ function listAt(data, path, firstOut, level, newest, cap = Number.POSITIVE_INFIN
410
663
  order.push(leavingOrder(j));
411
664
  const named = order.length;
412
665
  const queued = new Set(order);
413
- for (let j = 0; j < n; j++)
414
- if (!queued.has(leavingOrder(j)))
415
- order.push(leavingOrder(j));
666
+ const held = (i) => last !== undefined && last.has(v[i]);
667
+ for (const late of last ? [false, true] : [false])
668
+ for (let j = 0; j < n; j++)
669
+ if (!queued.has(leavingOrder(j)) && (!last || held(leavingOrder(j)) === late))
670
+ order.push(leavingOrder(j));
671
+ // #3269's second review: an order the cut chose (the rows kept by fit, cutData), named first.
672
+ if (ordered && ordered.length === n)
673
+ order.splice(0, n, ...ordered);
416
674
  // The characters of the entries in the order they leave, summed: leaving[j] is the first j's.
417
675
  let leaving;
676
+ const lensOf = () => {
677
+ if (cache.lens)
678
+ return cache.lens;
679
+ const lens = [];
680
+ for (let i = 0; i < n; i++)
681
+ lens.push(entryChars(i in made ? made[i] : cutOne(i)));
682
+ cache.lens = lens;
683
+ return lens;
684
+ };
418
685
  const sums = () => {
419
686
  if (leaving)
420
687
  return leaving;
421
- const lens = [];
422
- for (let i = 0; i < n; i++)
423
- lens.push(entryChars(!level || i in made ? entry(i) : thinned(v[i], level, cap)));
688
+ const ls = lensOf();
424
689
  leaving = [0];
425
690
  for (const i of order)
426
- leaving.push(leaving[leaving.length - 1] + lens[i]);
691
+ leaving.push(leaving[leaving.length - 1] + ls[i]);
427
692
  return leaving;
428
693
  };
429
694
  const kept = (k) => Math.max(0, Math.min(n, k));
@@ -444,6 +709,21 @@ function listAt(data, path, firstOut, level, newest, cap = Number.POSITIVE_INFIN
444
709
  ...(levelAt !== undefined ? { level: levelAt } : {}),
445
710
  cap,
446
711
  over,
712
+ ...(outLevel && named > 0 ? { outThin: levelOf(outLevel).keep } : {}),
713
+ isNamed,
714
+ ...(last ? { leaveLast: last } : {}),
715
+ othersOut: (k) => {
716
+ const out = order.slice(0, n - kept(k)).filter((i) => !isNamed(i));
717
+ const own = (i) => (last ? field(v[i], last.id) : undefined);
718
+ const byPosition = out.some((i) => typeof own(i) !== "number" || !Number.isFinite(own(i)));
719
+ const id = (i) => (byPosition ? i + 1 : own(i));
720
+ const asc = (xs) => xs.sort((a, b) => id(a) - id(b));
721
+ return { held: asc(out.filter(held)), rest: asc(out.filter((i) => !held(i))), id, byPosition };
722
+ },
723
+ ...(below ? { below } : {}),
724
+ order,
725
+ charsOf: (i) => lensOf()[i],
726
+ withOrder: (o) => listAt(data, path, firstOut, level, newest, cap, levelAt, outLevel, last, below, o, cache),
447
727
  chars: (k) => {
448
728
  const l = sums();
449
729
  return l[n] - l[n - kept(k)] + Math.max(0, kept(k) - 1);
@@ -484,14 +764,73 @@ function givenBackSaid(list, k) {
484
764
  const one = (r) => `${r.from === r.to ? `entry ${r.from}` : `entries ${r.from} to ${r.to}`} cut to ${keepSaid(r.g.keep)}${r.g.said ? `, ${r.g.said}` : ""}`;
485
765
  return `; with the room left, ${n} of the entries kept take back detail (by position among those kept, in their order in data): ${runs.map(one).join("; ")}`;
486
766
  }
767
+ // The most ids, or runs of them, the clause on the entries not named that left lists (#3269's
768
+ // second review): a client calls those again a few calls' worth at a time (workload_triage: at
769
+ // most 5 calls), and past this many it says how many more. #3269's third review: those LeaveLast
770
+ // holds and the others share it, the held ones first.
771
+ export const OTHERS_LEFT_LISTED = 100;
772
+ // Integers, ascending, as a sentence names them, runs of three or more as "a to b", at most `max`
773
+ // parts: the parts, how many integers they say (`n`) and leave out (`more`), and the last said.
774
+ const runsSaid = (xs, max = OTHERS_LEFT_LISTED) => {
775
+ const parts = [];
776
+ // xs[0..j) are said.
777
+ let j = 0;
778
+ while (j < xs.length && parts.length < max) {
779
+ let e = j;
780
+ while (e + 1 < xs.length && xs[e + 1] === xs[e] + 1)
781
+ e++;
782
+ if (e - j >= 2) {
783
+ parts.push(`${xs[j]} to ${xs[e]}`);
784
+ j = e + 1;
785
+ }
786
+ else {
787
+ parts.push(String(xs[j]));
788
+ j++;
789
+ }
790
+ }
791
+ return { said: listed(parts), parts: parts.length, n: j, more: xs.length - j, last: xs[j - 1] };
792
+ };
793
+ // The clause on the `g` entries not named that left, with LeaveLast: how many, the order they are
794
+ // kept in, and which (by its id), those LeaveLast holds first and apart (#3269's third review);
795
+ // and, where LeaveLast.again gives it, the sentence on checking them again: in that order, in calls
796
+ // of the sizes PageAt.againFit gives.
797
+ // With `counted` false the clause leaves out the count (the sentence has just said it). The order
798
+ // they were kept in is said where it shows (some entry kept after one that left, in input order).
799
+ function othersLeftSaid(list, k, g, fit, counted = true) {
800
+ const ll = list.leaveLast;
801
+ const { held, rest, id, byPosition } = list.othersOut(k);
802
+ const by = byPosition ? `position in ${list.path} (from 1)` : ll.id;
803
+ const h = runsSaid(held.map(id));
804
+ const r = runsSaid(rest.map(id), OTHERS_LEFT_LISTED - h.parts);
805
+ const at = (x) => x.n === 0 ? `not named here (the note names at most ${OTHERS_LEFT_LISTED} of them, or runs of them)` : `at ${by} ${x.said}${x.more > 0 ? `, and ${x.more} more past ${by} ${x.last}` : ""}`;
806
+ const which = held.length === 0
807
+ ? `, ${g === 1 ? "without" : "none with"} ${ll.said}, ${at(r)}`
808
+ : rest.length === 0
809
+ ? `, ${g === 1 ? "with" : "all with"} ${ll.said}, ${at(h)}`
810
+ : `: the ${held.length === 1 ? "one" : held.length} with ${ll.said} ${at(h)}; the other ${rest.length} ${at(r)}`;
811
+ // Where none of them is one LeaveLast holds and each left after every entry kept (the last
812
+ // ones), the order shows nothing to say (the synthetic's 201-purl probe: 43 rows kept with it
813
+ // unsaid, 42 with it said).
814
+ const kept = list.kept(k).filter((i) => !list.isNamed(i));
815
+ const last = held.length === 0 && (kept.length === 0 || rest.every((i) => i > kept[kept.length - 1]));
816
+ const said = `${counted ? `${g} ` : ""}${g === 1 ? ll.rest[0] : ll.rest[1]}${which}${last ? "" : ` (${ll.chosen})`}`;
817
+ if (!ll.again || byPosition)
818
+ return { said, again: "" };
819
+ // The entries the note names, in the order it names them.
820
+ const seq = [...held.slice(0, h.n), ...rest.slice(0, r.n)];
821
+ const { sizes, m, alone } = fit.againFit(list.path, seq);
822
+ const a = runsSaid(alone.map(id), Number.POSITIVE_INFINITY);
823
+ return { said, again: ll.again({ k: seq.length, sizes, m, alone: a.said, aloneCount: alone.length }) };
824
+ }
487
825
  // What a cut keeps of each list it has cut so far, as truncated and as the DATA CUT sentences.
488
- function cutSaid(cut, kept, wholeChars, data, over, wholeFit) {
826
+ function cutSaid(cut, kept, wholeChars, data, over, fit) {
489
827
  const isThinned = ({ list, k }) => list.thin !== undefined && k > 0;
490
828
  const cutOnes = kept.filter((x) => x.k < x.list.n || isThinned(x));
491
829
  if (cutOnes.length === 0)
492
830
  return { said: "" };
493
831
  const truncated = Object.fromEntries(cutOnes.map((x) => [x.list.path, { total: x.list.n, returned: x.k, ...(isThinned(x) ? { thinned: true } : {}) }]));
494
832
  const parts = [];
833
+ const again = [];
495
834
  const byKept = new Map();
496
835
  for (const x of cutOnes) {
497
836
  const { list, k } = x;
@@ -499,9 +838,29 @@ function cutSaid(cut, kept, wholeChars, data, over, wholeFit) {
499
838
  const first = Math.min(out, list.named);
500
839
  const end = list.newest ? "oldest" : "last";
501
840
  const which = list.firstOut ? `whose ${list.firstOut.field} is ${orListed(list.firstOut.values)} (${list.firstOut.why})` : "";
502
- const leftOut = out === 0 ? "" : first === 0 ? `, the ${end} ${out} left out` : first === out ? `, the ${out} left out being entries ${which}` : `, the ${out} left out being the ${first} ${which} and the ${end} ${out - first} of the rest`;
841
+ // #3269's second review: with LeaveLast, the entries not named that left are named by its id,
842
+ // so a client can call them alone again, and the order they leave in is said. #3269's third
843
+ // review: and how many of them, in the order named, a call of them alone keeps every one of.
844
+ const others = list.leaveLast && out > first ? othersLeftSaid(list, k, out - first, fit, first > 0) : undefined;
845
+ const rest = others?.said ?? "";
846
+ if (others?.again)
847
+ again.push(others.again);
848
+ const leftOut = out === 0
849
+ ? ""
850
+ : rest
851
+ ? first === 0
852
+ ? `, the ${out} left out being ${rest}`
853
+ : `, the ${out} left out being the ${first} ${which} and ${rest}`
854
+ : first === 0
855
+ ? `, the ${end} ${out} left out`
856
+ : first === out
857
+ ? `, the ${out} left out being entries ${which}`
858
+ : `, the ${out} left out being the ${first} ${which} and the ${end} ${out - first} of the rest`;
503
859
  if (isThinned(x)) {
504
- parts.push(`${list.path} keeps ${k === list.n ? `all ${k}` : `${k} of its ${list.n}`} entries, each cut to ${keepSaid(list.thin)}${list.thinSaid ? `, ${list.thinSaid}` : ""}${leftOut}${givenBackSaid(list, k)}`);
860
+ // #3269: the entries outFirst cut apart, those of them kept.
861
+ const apart = list.outThin ? list.kept(k).filter((i) => list.isNamed(i)).length : 0;
862
+ const apartSaid = apart > 0 && list.firstOut ? `, except the ${apart} kept whose ${list.firstOut.field} is ${orListed(list.firstOut.values)}, each cut to ${keepSaid(list.outThin)}` : "";
863
+ parts.push(`${list.path} keeps ${k === list.n ? `all ${k}` : `${k} of its ${list.n}`} entries, each cut to ${keepSaid(list.thin)}${list.thinSaid ? `, ${list.thinSaid}` : ""}${apartSaid}${leftOut}${givenBackSaid(list, k)}`);
505
864
  continue;
506
865
  }
507
866
  if (!list.firstOut || first === 0 || k === 0) {
@@ -529,9 +888,10 @@ function cutSaid(cut, kept, wholeChars, data, over, wholeFit) {
529
888
  return p.k === 0 ? `${paths} keep none of their entries` : `${paths} keep their ${which} ${p.k} entries each`;
530
889
  });
531
890
  const keeps = Object.fromEntries(cutOnes.filter(isThinned).map((x) => [x.list.path, x.list.thin]));
532
- const page = cut.page?.(truncated, data, { keeps, wholeFit }) ?? "";
891
+ const page = cut.page?.(truncated, data, { keeps, ...fit }) ?? "";
533
892
  const said = [
534
893
  `DATA CUT: with data whole, this structured result is ${wholeChars} characters of JSON, more than the ${STRUCTURED_BUDGET} it holds unless the call says full: true, so its data is cut: ${cuts.join("; ")}.`,
894
+ ...again,
535
895
  "truncated gives the total and the number returned of each list cut, and thinned where its entries are cut to some of their fields.",
536
896
  page,
537
897
  over ? `Cut so, the structured result is still past ${STRUCTURED_BUDGET} characters: data holds no other list to cut.` : "",
@@ -552,6 +912,8 @@ export function cutData(data, cut, wholeChars, structured) {
552
912
  const steps = [...cut.steps, { lists: Object.keys(data).filter((k) => !named.has(k)) }];
553
913
  // Each list cut so far, in the order cut, at the level it was thinned to, how many entries it
554
914
  // keeps, and the step that cut it.
915
+ // `gone` (#3269): the list's entries outFirst names left before the others were cut further, and
916
+ // may come back once the cut is settled.
555
917
  const kept = new Map();
556
918
  const apply = (also = []) => [...kept.values(), ...also].reduce((d, { list, k }) => withList(d, list.at, list.keep(k)), data);
557
919
  // PageAt.wholeFit: of the list at `path`, the most first entries, each whole, that the result
@@ -577,13 +939,244 @@ export function cutData(data, cut, wholeChars, structured) {
577
939
  wholeFits.set(path, k);
578
940
  return k;
579
941
  };
942
+ // PageAt.firstFit (#3269's review): of the list at `path`, which a step thins, the most first
943
+ // entries n such that, for every n' up to n, a call answering only the first n' of them keeps
944
+ // every one of them, as this cut would cut that call: it does where
945
+ // - every entry fits thinned (thin[0]), those leaveOutFirst names cut to outFirst's level; or
946
+ // - every entry fits at the floor (thin[floor.at] at its least, each entry at its own
947
+ // EntryLevel.floor or more: #3269's second review), those named so cut, and either none is
948
+ // named or the entries not named, thinned, are past the budget on their own (so the call
949
+ // cannot keep them uncapped by sending the named ones out, and keeps them at the floor
950
+ // instead).
951
+ // A step without a floor is held to its leaveAt level and cap instead, so a page of that size
952
+ // comes back with that level's detail, as #3132's review asked. Each size is counted as this
953
+ // call's: the entries as they are (the first n of a call are these, at the same index), data
954
+ // beside them with this list and the lists of the steps before it empty (a cut empties those
955
+ // first, and a call of fewer purls sends them all), and this call's envelope and note, with the
956
+ // longest DATA CUT sentence such a call can carry while it keeps every entry. A call of fewer
957
+ // purls has a note no longer than this one's (its counts are no larger, the affected components
958
+ // it names are the first of these), but for its TEXT CUT sentences, which can be cut no further
959
+ // than this call's text but can take another form: FIRST_FIT_SLACK holds the difference.
960
+ const FIRST_FIT_SLACK = 600;
961
+ const measures = new Map();
962
+ const measureOf = (path) => {
963
+ const known = measures.get(path);
964
+ if (known !== undefined)
965
+ return known;
966
+ const s = steps.findIndex((st) => st.lists.includes(path));
967
+ const v = listOf(data, path.split("."));
968
+ const m = s >= 0 && steps[s].thin?.length && Array.isArray(v) && v.length > 0 ? measureFor(steps[s], s, path, v) : null;
969
+ measures.set(path, m);
970
+ return m;
971
+ };
972
+ const measureFor = (step, s, path, v) => {
973
+ const thin = step.thin;
974
+ const first = step.leaveOutFirst;
975
+ const out = first ? step.outFirst : undefined;
976
+ const floor = out?.floor;
977
+ const lean = out?.level;
978
+ const n = v.length;
979
+ const isNamed = (e) => first !== undefined && first.values.includes(field(e, first.field));
980
+ const bare = [path, ...steps.slice(0, s).flatMap((st) => st.lists)].reduce((d, p) => (Array.isArray(listOf(d, p.split("."))) ? withList(d, p.split("."), []) : d), data);
981
+ const base = JSON.stringify(bare).length;
982
+ // The level and cap every entry is held to where they are not all kept thinned.
983
+ const atFloor = floor ? floor.at : step.leaveAt ? step.leaveAt.level : thin.length - 1;
984
+ const floorCap = floor ? (levelOf(thin[atFloor]).least ?? 0) : step.leaveAt ? step.leaveAt.cap : Number.POSITIVE_INFINITY;
985
+ // The envelope, its note and the longest DATA CUT sentence of a call that keeps every entry,
986
+ // at each level it can stop at (a capped level's clause said at its least, where it bites).
987
+ const noPage = { steps: cut.steps };
988
+ const stops = [0, ...(floor && out ? Array.from({ length: floor.at - out.before + 1 }, (_, j) => out.before + j) : [atFloor])];
989
+ const env = Math.max(...stops.map((j) => {
990
+ const lv = levelOf(thin[j]);
991
+ const list = listAt(data, path, first, thin[j], step.keepNewest, lv.capped ? (lv.least ?? 0) : Number.POSITIVE_INFINITY, j, lean, step.leaveLast);
992
+ if (!list)
993
+ return 0;
994
+ const { said } = cutSaid(noPage, [{ list, k: n }], wholeChars, data, false, { wholeFit, firstFit: () => 0, againFit: () => ({ sizes: [], m: 0, alone: [] }) });
995
+ return JSON.stringify(structured(null, { [path]: { total: n, returned: n, thinned: true } }, said)).length - "null".length;
996
+ })) +
997
+ base +
998
+ FIRST_FIT_SLACK;
999
+ const chars = (e, level, cap = Number.POSITIVE_INFINITY) => entryChars(thinned(e, level, cap));
1000
+ const once = (f) => {
1001
+ const got = new Map();
1002
+ return (i) => {
1003
+ let c = got.get(i);
1004
+ if (c === undefined)
1005
+ got.set(i, (c = f(i)));
1006
+ return c;
1007
+ };
1008
+ };
1009
+ const named = (i) => isNamed(v[i]);
1010
+ return {
1011
+ v,
1012
+ step,
1013
+ env,
1014
+ named,
1015
+ thinChars: once((i) => chars(v[i], named(i) && lean !== undefined ? lean : thin[0])),
1016
+ floorChars: once((i) => (named(i) && lean !== undefined ? chars(v[i], lean) : chars(v[i], thin[atFloor], floorCap))),
1017
+ };
1018
+ };
1019
+ const firstFits = new Map();
1020
+ const firstFit = (path) => {
1021
+ const known = firstFits.get(path);
1022
+ if (known !== undefined)
1023
+ return known;
1024
+ const ms = measureOf(path);
1025
+ const fit = ms ? firstFitOf(ms) : 0;
1026
+ firstFits.set(path, fit);
1027
+ return fit;
1028
+ };
1029
+ const firstFitOf = ({ v, step, env, named: isNamed, thinChars, floorChars }) => {
1030
+ const out = step.leaveOutFirst ? step.outFirst : undefined;
1031
+ const floor = out?.floor;
1032
+ const n = v.length;
1033
+ let thinnedAll = 0;
1034
+ let atFloorAll = 0;
1035
+ let others = 0;
1036
+ let othersAlone = 0;
1037
+ let named = 0;
1038
+ for (let i = 0; i < n; i++) {
1039
+ const nm = isNamed(i);
1040
+ if (nm)
1041
+ named++;
1042
+ thinnedAll += thinChars(i);
1043
+ atFloorAll += floorChars(i);
1044
+ if (!nm) {
1045
+ others++;
1046
+ othersAlone += thinChars(i);
1047
+ }
1048
+ const commas = i;
1049
+ const keepsThinned = env + thinnedAll + commas <= STRUCTURED_BUDGET;
1050
+ const keepsAtFloor = env + atFloorAll + commas <= STRUCTURED_BUDGET;
1051
+ // The entries not named, thinned, without the rest of the result: a lower bound.
1052
+ const othersPast = othersAlone + Math.max(0, others - 1) + 2 > STRUCTURED_BUDGET;
1053
+ if (!(keepsThinned || (keepsAtFloor && (named === 0 || !out || (floor !== undefined && othersPast)))))
1054
+ return i;
1055
+ }
1056
+ return n;
1057
+ };
1058
+ // PageAt.againFit (#3269's third review): of the entries `seq` names (none of them one
1059
+ // leaveOutFirst names, so a call of them alone keeps every one where they fit at their floor, as
1060
+ // firstFit counts), those that do not fit at it even alone leave `seq` (`alone`). Of the rest,
1061
+ // each of the first LeaveLast.againCalls calls takes the most of those left, in order, that fit
1062
+ // at the floor beside the result's envelope (this call's, its note with LeaveLast.noteOver's
1063
+ // characters for a call of that many, and a DATA CUT sentence keeping them all); of the entries
1064
+ // after those, m is the most such that every m of them in a row fit so. So each call of those
1065
+ // sizes keeps every one, and so does a call of fewer of them in a row. Counted once per `seq`
1066
+ // (the cut sizes the same note many times).
1067
+ const againFits = new Map();
1068
+ const againFit = (path, seq) => {
1069
+ const key = `${path}:${seq.join(",")}`;
1070
+ const known = againFits.get(key);
1071
+ if (known)
1072
+ return known;
1073
+ const got = againFitOf(path, seq);
1074
+ againFits.set(key, got);
1075
+ return got;
1076
+ };
1077
+ const againFitOf = (path, seq) => {
1078
+ const ms = measureOf(path);
1079
+ if (!ms || seq.length === 0)
1080
+ return { sizes: [], m: 0, alone: [] };
1081
+ const ll = ms.step.leaveLast;
1082
+ const over = ll?.noteOver?.(seq.map((i) => ms.v[i]), ms.v) ?? (() => 0);
1083
+ const room = (m) => STRUCTURED_BUDGET - ms.env - over(m);
1084
+ const one = room(1);
1085
+ const alone = seq.filter((i) => ms.floorChars(i) > one);
1086
+ const rest = alone.length ? seq.filter((i) => ms.floorChars(i) <= one) : seq;
1087
+ const calls = ll?.againCalls ?? 0;
1088
+ // sums[j]: the characters of rest's first j entries; rest[a..b) fits as one call.
1089
+ const sums = [0];
1090
+ for (const i of rest)
1091
+ sums.push(sums[sums.length - 1] + ms.floorChars(i));
1092
+ const fitsAt = (a, b) => sums[b] - sums[a] + (b - a - 1) <= room(b - a);
1093
+ const sizes = [];
1094
+ let at = 0;
1095
+ while (sizes.length < calls && at < rest.length) {
1096
+ let b = at + 1;
1097
+ while (b < rest.length && fitsAt(at, b + 1))
1098
+ b++;
1099
+ sizes.push(b - at);
1100
+ at = b;
1101
+ }
1102
+ // Every m of rest[at..] in a row fits (fewer, at its end, are within the last m).
1103
+ const fits = (m) => {
1104
+ for (let a = at; a + m <= rest.length; a++)
1105
+ if (!fitsAt(a, a + m))
1106
+ return false;
1107
+ return true;
1108
+ };
1109
+ let m = rest.length > at ? 1 : 0;
1110
+ let hi = rest.length - at;
1111
+ while (m < hi) {
1112
+ const mid = Math.ceil((m + hi) / 2);
1113
+ if (fits(mid))
1114
+ m = mid;
1115
+ else
1116
+ hi = mid - 1;
1117
+ }
1118
+ return { sizes, m, alone };
1119
+ };
1120
+ const fit = { wholeFit, firstFit, againFit };
580
1121
  // The result's size with this step's lists at k entries each: the rest of the result (data
581
1122
  // null), data with this step's lists empty (`base`), and their entries.
582
1123
  const size = (lists, k, base) => {
583
1124
  const all = [...kept.values(), ...lists.map((list) => ({ list, k: Math.min(k, list.n) }))];
584
- const { truncated, said } = cutSaid(cut, all, wholeChars, data, false, wholeFit);
1125
+ const { truncated, said } = cutSaid(cut, all, wholeChars, data, false, fit);
585
1126
  return JSON.stringify(structured(null, truncated, said)).length - "null".length + base + lists.reduce((t, l) => t + l.chars(k), 0);
586
1127
  };
1128
+ // #3269's second review: whether an entry of a step's lists that leaveOutFirst does not name holds
1129
+ // any entry of the list leaveAt's level caps (DataCutStep.leaveAt's floor).
1130
+ const holdsInner = (step) => {
1131
+ const inner = step.thin && step.leaveAt ? levelOf(step.thin[step.leaveAt.level]).capped : undefined;
1132
+ if (inner === undefined)
1133
+ return false;
1134
+ const first = step.leaveOutFirst;
1135
+ const named = (e) => first !== undefined && first.values.includes(field(e, first.field));
1136
+ return step.lists.some((p) => {
1137
+ const v = listOf(data, p.split("."));
1138
+ return Array.isArray(v) && v.some((e) => !named(e) && innerOf(e, inner) > 0);
1139
+ });
1140
+ };
1141
+ // #3269's second review: with leaveAt's floor and LeaveLast, the entries kept are chosen in the
1142
+ // reverse of the order they leave in (LeaveLast's first, then the rest, each from the start),
1143
+ // each one kept that fits beside those chosen before it, and the rest leave first: the list so
1144
+ // reordered, and how many it keeps. In the plain order a row too large to fit beside the rows kept
1145
+ // sent every row before it in that order out with it, and left their room empty: a row of 200
1146
+ // KEV-listed matches after 150 purls of an image (40 affected) kept 1 row of 151, in 7,572
1147
+ // characters. The choice is made on the room the result leaves with none kept, then checked as
1148
+ // the result is (its note names the entries left out, by index), the last chosen leaving while it
1149
+ // does not fit, and topped up, each entry not chosen that fits now, in the same order.
1150
+ const keptByFit = (l, base) => {
1151
+ if (size([l], 0, base) > STRUCTURED_BUDGET)
1152
+ return undefined;
1153
+ const named = l.order.filter((i) => l.isNamed(i));
1154
+ const pri = [...l.order].reverse().filter((i) => !l.isNamed(i));
1155
+ const orderOf = (chosen) => [...named, ...l.order.filter((i) => !l.isNamed(i) && !chosen.has(i)), ...l.order.filter((i) => chosen.has(i))];
1156
+ const chosen = new Set();
1157
+ let room = STRUCTURED_BUDGET - size([l], 0, base);
1158
+ for (const i of pri) {
1159
+ const c = l.charsOf(i) + (chosen.size > 0 ? 1 : 0);
1160
+ if (c > room)
1161
+ continue;
1162
+ chosen.add(i);
1163
+ room -= c;
1164
+ }
1165
+ let cur = l.withOrder(orderOf(chosen));
1166
+ let k = chosen.size;
1167
+ while (k > 0 && size([cur], k, base) > STRUCTURED_BUDGET)
1168
+ k--;
1169
+ for (let tries = 0; tries < 64; tries++) {
1170
+ const kept = new Set(cur.kept(k));
1171
+ const left = STRUCTURED_BUDGET - size([cur], k, base);
1172
+ const next = pri.find((i) => !kept.has(i) && cur.charsOf(i) + 1 <= left && size([cur.withOrder(orderOf(new Set([...kept, i])))], k + 1, base) <= STRUCTURED_BUDGET);
1173
+ if (next === undefined)
1174
+ break;
1175
+ cur = cur.withOrder(orderOf(new Set([...kept, next])));
1176
+ k++;
1177
+ }
1178
+ return { list: cur, k };
1179
+ };
587
1180
  let fits = false;
588
1181
  for (const [s, step] of steps.entries()) {
589
1182
  // Every entry whole, then each thinning level in turn, every entry kept; at the leanest level
@@ -591,13 +1184,105 @@ export function cutData(data, cut, wholeChars, structured) {
591
1184
  const levels = [undefined, ...(step.thin ?? [])];
592
1185
  let lists = [];
593
1186
  let k = 0;
594
- const at = (level, cap, levelAt) => step.lists.map((p) => listAt(data, p, step.leaveOutFirst, level, step.keepNewest, cap, levelAt)).filter((l) => l !== undefined && !kept.has(l.path));
1187
+ // #3269: once the entries outFirst names have left (gone), each later level cuts them to its
1188
+ // level, so that they can come back so cut, and keeps at most the others (they leave first).
1189
+ const out = step.leaveOutFirst ? step.outFirst : undefined;
1190
+ let gone = false;
1191
+ const at = (level, cap, levelAt, outLevel = gone ? out?.level : undefined, below) => step.lists
1192
+ .map((p) => listAt(data, p, step.leaveOutFirst, level, step.keepNewest, cap, levelAt, outLevel, step.leaveLast, below))
1193
+ .filter((l) => l !== undefined && !kept.has(l.path));
1194
+ // #3269's second review: with leaveAt's floor, the last level tried with every entry kept is
1195
+ // leaveAt's while an entry not named holds inner entries (DataCutStep.leaveAt): no later level
1196
+ // keeps any of them, so the entries leave there, whole, instead.
1197
+ const end = step.leaveAt?.floor && step.thin && holdsInner(step) ? step.leaveAt.level + 1 : levels.length - 1;
595
1198
  for (const [i, thin] of levels.entries()) {
1199
+ if (i > end)
1200
+ break;
1201
+ // #3269: before any entry is cut to thin[out.before], the entries leaveOutFirst names are cut
1202
+ // to out.level, the others kept at the level before; then they leave, as many as must.
1203
+ if (out && !gone && i === out.before + 1) {
1204
+ const apart = at(levels[i - 1], undefined, i - 2, out.level);
1205
+ if (apart.some((l) => l.named > 0)) {
1206
+ const base = JSON.stringify(apply(apart.map((list) => ({ list, k: 0 })))).length;
1207
+ const rest = Math.max(...apart.map((l) => l.n - l.named));
1208
+ const all = Math.max(...apart.map((l) => l.n));
1209
+ // The most entries each list keeps, `lo` at least, cut as `ls` is.
1210
+ const most = (ls, lo) => {
1211
+ let kk = lo;
1212
+ let hi = all;
1213
+ while (kk < hi) {
1214
+ const mid = Math.ceil((kk + hi) / 2);
1215
+ if (size(ls, mid, base) <= STRUCTURED_BUDGET)
1216
+ kk = mid;
1217
+ else
1218
+ hi = mid - 1;
1219
+ }
1220
+ return kk;
1221
+ };
1222
+ if (size(apart, rest, base) <= STRUCTURED_BUDGET) {
1223
+ k = most(apart, rest);
1224
+ lists = apart;
1225
+ fits = true;
1226
+ break;
1227
+ }
1228
+ // #3269's review: the others are capped anyway, so the named entries stay while the
1229
+ // others fit at the floor (OutFirst.floor). #3269's second review: each capped level
1230
+ // holds each entry's own floor (EntryLevel.floor), so the cap starts at its least.
1231
+ const floor = out.floor;
1232
+ if (floor) {
1233
+ const last = floor.at + 1;
1234
+ // With kk entries kept, the richest capped level from thin[before] to thin[floor.at]
1235
+ // that fits at the floor or more, at the most inner entries that fit; none where none.
1236
+ const capped = (kk) => {
1237
+ for (let j = i; j <= last; j++) {
1238
+ const lv = levelOf(levels[j]);
1239
+ if (lv.capped === undefined)
1240
+ continue;
1241
+ const ls = (c) => at(levels[j], c, j - 1, out.level);
1242
+ let m = lv.least ?? 0;
1243
+ if (size(ls(m), kk, base) > STRUCTURED_BUDGET)
1244
+ continue;
1245
+ let hi = Math.max(m, ...step.lists.map((p) => longestInside(listOf(data, p.split(".")), lv.capped)));
1246
+ while (m < hi) {
1247
+ const mid = Math.ceil((m + hi) / 2);
1248
+ if (size(ls(mid), kk, base) <= STRUCTURED_BUDGET)
1249
+ m = mid;
1250
+ else
1251
+ hi = mid - 1;
1252
+ }
1253
+ return ls(m);
1254
+ }
1255
+ return undefined;
1256
+ };
1257
+ let kk = all;
1258
+ let got = capped(all);
1259
+ if (!got) {
1260
+ // Even the floor does not fit with every named entry: they leave, as many as must
1261
+ // for it to, and the others then take the most that fits.
1262
+ const lv = levelOf(levels[last]);
1263
+ const atLeast = at(levels[last], lv.least ?? 0, last - 1, out.level);
1264
+ if (size(atLeast, rest, base) <= STRUCTURED_BUDGET) {
1265
+ kk = most(atLeast, rest);
1266
+ got = capped(kk);
1267
+ }
1268
+ }
1269
+ if (got) {
1270
+ lists = got;
1271
+ k = kk;
1272
+ if (kk < all)
1273
+ gone = true;
1274
+ fits = true;
1275
+ break;
1276
+ }
1277
+ }
1278
+ gone = true;
1279
+ }
1280
+ }
596
1281
  lists = at(thin, undefined, i - 1);
597
1282
  if (lists.length === 0)
598
1283
  break;
599
1284
  const base = JSON.stringify(apply(lists.map((list) => ({ list, k: 0 })))).length;
600
- const top = Math.max(...lists.map((l) => l.n));
1285
+ const top = Math.max(...lists.map((l) => (gone ? l.n - l.named : l.n)));
601
1286
  // #3132: a level that caps a list inside each entry, every entry kept: the most entries of
602
1287
  // that inner list, the same number in each, that fit; a level where even none fits is passed.
603
1288
  // #3132's review: a level with `least` is taken at that cap or more, or passed.
@@ -620,12 +1305,12 @@ export function cutData(data, cut, wholeChars, structured) {
620
1305
  fits = true;
621
1306
  break;
622
1307
  }
623
- if (i < levels.length - 1)
1308
+ if (i < end)
624
1309
  continue;
625
1310
  // The last level, and even its least cap does not fit: the entries go on at that cap.
626
1311
  lists = at(thin, least, i - 1);
627
1312
  }
628
- if (i < levels.length - 1) {
1313
+ if (i < end) {
629
1314
  if (size(lists, top, base) <= STRUCTURED_BUDGET) {
630
1315
  k = top;
631
1316
  fits = true;
@@ -652,7 +1337,13 @@ export function cutData(data, cut, wholeChars, structured) {
652
1337
  }
653
1338
  break;
654
1339
  }
655
- if (size(lists, 0, base) <= STRUCTURED_BUDGET) {
1340
+ const chosen = step.leaveAt?.floor && step.leaveLast && lists.length === 1 ? keptByFit(lists[0], base) : undefined;
1341
+ if (chosen) {
1342
+ lists = [chosen.list];
1343
+ k = chosen.k;
1344
+ fits = true;
1345
+ }
1346
+ else if (size(lists, 0, base) <= STRUCTURED_BUDGET) {
656
1347
  // The most entries each list of the step keeps with the result inside the budget.
657
1348
  let hi = top;
658
1349
  while (k < hi) {
@@ -664,14 +1355,47 @@ export function cutData(data, cut, wholeChars, structured) {
664
1355
  }
665
1356
  fits = true;
666
1357
  }
1358
+ // #3269's second review: not even one entry fits at its floor (one extreme row, its KEV-listed
1359
+ // matches alone past the budget). The one the leaving order keeps last stays, its capped list
1360
+ // at the most that fit below its floor, at least one; where not even one fits, at the first
1361
+ // later level that fits; and only where none does, no entry.
1362
+ if (fits && k === 0 && top > 0 && step.leaveAt?.floor && step.thin) {
1363
+ const { level } = step.leaveAt;
1364
+ const inner = levelOf(step.thin[level]).capped;
1365
+ let got;
1366
+ if (inner !== undefined) {
1367
+ const ls = (c) => at(step.thin?.[level], c, level, undefined, true);
1368
+ if (size(ls(1), 1, base) <= STRUCTURED_BUDGET) {
1369
+ let c = 1;
1370
+ let hi = Math.max(1, ...step.lists.map((p) => longestInside(listOf(data, p.split(".")), inner))) - 1;
1371
+ while (c < hi) {
1372
+ const mid = Math.ceil((c + hi) / 2);
1373
+ if (size(ls(mid), 1, base) <= STRUCTURED_BUDGET)
1374
+ c = mid;
1375
+ else
1376
+ hi = mid - 1;
1377
+ }
1378
+ got = ls(c);
1379
+ }
1380
+ }
1381
+ for (let j = level + 1; !got && j < step.thin.length; j++) {
1382
+ const ls = at(step.thin[j], undefined, j);
1383
+ if (size(ls, 1, base) <= STRUCTURED_BUDGET)
1384
+ got = ls;
1385
+ }
1386
+ if (got) {
1387
+ lists = got;
1388
+ k = 1;
1389
+ }
1390
+ }
667
1391
  }
668
1392
  for (const list of lists)
669
- kept.set(list.path, { list, k: Math.min(k, list.n), step: s });
1393
+ kept.set(list.path, { list, k: Math.min(k, list.n), step: s, ...(gone ? { gone: true } : {}) });
670
1394
  if (fits)
671
1395
  break;
672
1396
  }
673
1397
  const exact = () => {
674
- const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, false, wholeFit);
1398
+ const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, false, fit);
675
1399
  return JSON.stringify(structured(apply(), truncated, said)).length;
676
1400
  };
677
1401
  // #3139: every step cut and the result still past the budget: a keepAll step's entries leave now,
@@ -759,7 +1483,8 @@ export function cutData(data, cut, wholeChars, structured) {
759
1483
  for (const e of [...kept.values()].filter((x) => x.step === s)) {
760
1484
  const { list } = e;
761
1485
  const own = list.level;
762
- if (own === undefined || list.thin === undefined)
1486
+ // #3269's second review: the one entry kept below its floor takes back nothing (more did not fit).
1487
+ if (own === undefined || list.thin === undefined || list.below)
763
1488
  continue;
764
1489
  const whole = listOf(data, list.at);
765
1490
  if (!Array.isArray(whole))
@@ -777,13 +1502,16 @@ export function cutData(data, cut, wholeChars, structured) {
777
1502
  return false;
778
1503
  };
779
1504
  for (const i of list.kept(e.k)) {
1505
+ // #3269: an entry outFirst cut apart takes back nothing (the others come first).
1506
+ if (list.outThin && list.isNamed(i))
1507
+ continue;
780
1508
  // An entry its own level already keeps as the richest would is passed, not an end.
781
1509
  if (JSON.stringify(thinned(whole[i], thin[0])) === JSON.stringify(thinned(whole[i], thin[own], list.cap)))
782
1510
  continue;
783
1511
  let took = false;
784
1512
  for (let j = 0; j <= own && !took; j++) {
785
1513
  const l = levelOf(thin[j]);
786
- const inner = l.capped !== undefined && Array.isArray(field(whole[i], l.capped)) ? field(whole[i], l.capped).length : 0;
1514
+ const inner = innerOf(whole[i], l.capped);
787
1515
  if (l.capped === undefined) {
788
1516
  if (j < own)
789
1517
  took = tryAt(i, j, Number.POSITIVE_INFINITY);
@@ -791,11 +1519,13 @@ export function cutData(data, cut, wholeChars, structured) {
791
1519
  }
792
1520
  if (j !== leanestCapped)
793
1521
  continue;
794
- if (j < own || list.cap < inner)
1522
+ // #3269's second review: the cap the entry is cut to, its floor where that is more.
1523
+ const now = capFor(l, whole[i], list.cap);
1524
+ if (j < own || now < inner)
795
1525
  took = tryAt(i, j, Number.POSITIVE_INFINITY);
796
1526
  if (took)
797
1527
  break;
798
- let c = j === own ? list.cap + 1 : (l.least ?? 0);
1528
+ let c = j === own ? now + 1 : capFor(l, whole[i], l.least ?? 0);
799
1529
  let hi = inner - 1;
800
1530
  if (c > hi || !tryAt(i, j, c))
801
1531
  continue;
@@ -812,7 +1542,27 @@ export function cutData(data, cut, wholeChars, structured) {
812
1542
  }
813
1543
  }
814
1544
  }
815
- const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, !fits, wholeFit);
1545
+ // #3269: last, the entries outFirst sent out come back with the room left, cut to its level, the
1546
+ // last to leave first (they leave first of all, so they come back only where every other entry
1547
+ // stayed), as many as fit.
1548
+ if (fits) {
1549
+ for (const e of kept.values()) {
1550
+ if (!e.gone || e.k < e.list.n - e.list.named)
1551
+ continue;
1552
+ let hi = e.list.n;
1553
+ let k = e.k;
1554
+ while (k < hi) {
1555
+ const mid = Math.ceil((k + hi) / 2);
1556
+ e.k = mid;
1557
+ if (exact() <= STRUCTURED_BUDGET)
1558
+ k = mid;
1559
+ else
1560
+ hi = mid - 1;
1561
+ }
1562
+ e.k = k;
1563
+ }
1564
+ }
1565
+ const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, !fits, fit);
816
1566
  if (truncated === undefined)
817
1567
  return undefined;
818
1568
  const d = apply();