echelongraph-mcp 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +695 -2
- package/README.md +85 -74
- package/dist/fixStatus.js +4 -2
- package/dist/http.js +73 -81
- package/dist/httpPolicy.js +213 -11
- package/dist/index.js +516 -52
- package/dist/logLines.js +86 -0
- package/dist/prompts.js +126 -21
- package/dist/readBody.js +140 -0
- package/dist/requiredText.js +38 -3
- package/dist/resources.js +3 -3
- package/dist/runtime.js +66 -1
- package/dist/schemaRefusal.js +138 -0
- package/dist/textBudget.js +1082 -78
- package/dist/tools/check_affected.js +23 -7
- package/dist/tools/check_sbom.js +287 -8
- package/dist/tools/cve_intel.js +163 -8
- package/dist/tools/cve_remediation.js +174 -18
- package/dist/tools/epss_history.js +45 -6
- package/dist/tools/kev_recent.js +4 -1
- package/dist/tools/match_batch.js +78 -15
- package/dist/tools/packageRecords.js +58 -0
- package/dist/tools/scan_manifest.js +26 -7
- package/dist/tools/vendor_advisories.js +40 -8
- package/docs/exposure-radar.md +4 -0
- package/docs/rate-limits.md +6 -3
- package/docs/structured-results.md +15 -3
- package/docs/tools.md +149 -12
- package/package.json +1 -1
- package/server.json +2 -2
package/dist/textBudget.js
CHANGED
|
@@ -32,9 +32,20 @@ import * as z from "zod";
|
|
|
32
32
|
// in "…", every list in it to its first `cap` entries); then, for a tool that names the
|
|
33
33
|
// lists beside its rows (`sides`: check_affected's excluded sample, check_sbom's
|
|
34
34
|
// not_sent_purls), the same level with those lists cut as it says; then each next level with
|
|
35
|
-
// them cut; and when no level fits,
|
|
36
|
-
//
|
|
37
|
-
//
|
|
35
|
+
// them cut; and when no level fits, the lists beside the rows that the tool names
|
|
36
|
+
// beforeRows (#3272: cve_remediation's fixed_branches, patches and fix_references, which its
|
|
37
|
+
// structured cut thins and leaves in the same order) are cut as each says, one after another,
|
|
38
|
+
// then (3.1.0's phase A review, `capped`: fixed_branches) each entry's own list is cut to its
|
|
39
|
+
// first c entries, the most that fit, every entry kept and marked as the structured cut
|
|
40
|
+
// marks it, and then they lose entries from the end, the last named first, the room left
|
|
41
|
+
// going back to c; then rows are left out of the
|
|
42
|
+
// text, those the tool names first, then from the end, those it names last (leaveOutLast,
|
|
43
|
+
// #2899: check_affected's KEV-listed and ransomware matches) only after every other, the note
|
|
44
|
+
// naming each of those left out. A list the tool names beside its rows is cut before any row
|
|
45
|
+
// leaves the text, and, unless it is named beforeRows, before any row field the first level
|
|
46
|
+
// keeps (a list it does not name is cut only in step 4).
|
|
47
|
+
// With outFirstBefore (#3269: check_sbom), the rows the tool names are cut to the leanest
|
|
48
|
+
// level and leave first, as many as must, before any other row is cut to the level it names;
|
|
38
49
|
// 4. else every list keeps its first n entries, n halving, and then strings are cut too.
|
|
39
50
|
// Whenever the text is not the JSON whole, the note ends with one sentence, starting TEXT CUT,
|
|
40
51
|
// that says what the first text block leaves out, where structuredContent.data carries all of it,
|
|
@@ -62,7 +73,21 @@ import * as z from "zod";
|
|
|
62
73
|
// the tool names leave first (the clean rows, counted in data.summary), then the last of the
|
|
63
74
|
// rest, or for a step of series named keepNewest (exposure_radar's daily and weekly counts,
|
|
64
75
|
// which the API sends oldest first) its last ones, the newest, the oldest leaving first; a
|
|
65
|
-
// step that cannot fit even empty is emptied, and the next one is cut
|
|
76
|
+
// step that cannot fit even empty is emptied, and the next one is cut. A level can cut inside
|
|
77
|
+
// each entry on its own terms (an EntryLevel, #3132: check_sbom's rows keep at most the same
|
|
78
|
+
// number of matches, the most that fit, so only the rows with more lose any), and a step named
|
|
79
|
+
// keepAll (#3139: cve_remediation's vendor_remediations) keeps every entry at its leanest level
|
|
80
|
+
// while the later steps are cut, its entries leaving only once they all are; and a step named
|
|
81
|
+
// with outFirst (#3269: check_sbom's results) sends the entries leaveOutFirst names out, cut
|
|
82
|
+
// lean, before the others are cut past a given level, and has them back with the room left,
|
|
83
|
+
// but (#3269's review, outFirst.floor) keeps them while the others, capped anyway, fit at a
|
|
84
|
+
// floor; and a tool that pages names the page size at which a call of its first entries keeps
|
|
85
|
+
// them all (PageAt.firstFit), counted as this cut would cut that call; #3269's second review:
|
|
86
|
+
// with EntryLevel.floor and leaveAt's floor no entry's list goes below its floor, the entries
|
|
87
|
+
// leave whole instead, chosen by fit, those DataCutStep.leaveLast holds kept first, and the
|
|
88
|
+
// note names by id those of them that left (#3269's third review: those it holds first and
|
|
89
|
+
// apart, and, where the tool gives the sentence, the sizes of calls of them alone that keep
|
|
90
|
+
// every one, PageAt.againFit);
|
|
66
91
|
// 2. then every other list at the top level of data, the same way. A list inside an object of
|
|
67
92
|
// data (get_cve's remediation summary, with fix_status, #2841) is cut only where a DataCut
|
|
68
93
|
// names it, and none names get_cve's. Once a step fits, the room it leaves goes back to the
|
|
@@ -70,7 +95,9 @@ import * as z from "zod";
|
|
|
70
95
|
// emptied while its rows were whole, keeps its entries once the thinned rows fit);
|
|
71
96
|
// 3. no string is shortened and no value changes: a thinned entry keeps some of its fields as
|
|
72
97
|
// they are (every field a thinning level drops is optional in the outputSchema, so data still
|
|
73
|
-
// validates against it), and no field outside the lists cut changes
|
|
98
|
+
// validates against it), and no field outside the lists cut changes (the one field a cut
|
|
99
|
+
// adds is an EntryLevel's mark, check_sbom's matches_truncated, which says what the row's own
|
|
100
|
+
// cut left out, and which its outputSchema names); the envelope says
|
|
74
101
|
// data_truncated true, truncated gives each list cut by its path in data with its total, the
|
|
75
102
|
// number returned and, where its entries are thinned, thinned true, and the note ends with
|
|
76
103
|
// sentences starting DATA CUT that say what was cut and to which fields, how to read the rest
|
|
@@ -118,11 +145,12 @@ function cutValue(v, clip, cap, s) {
|
|
|
118
145
|
if (typeof v === "string") {
|
|
119
146
|
if (v.length <= clip)
|
|
120
147
|
return v;
|
|
121
|
-
s
|
|
148
|
+
if (s)
|
|
149
|
+
s.clipped++;
|
|
122
150
|
return clipString(v, clip);
|
|
123
151
|
}
|
|
124
152
|
if (Array.isArray(v)) {
|
|
125
|
-
if (v.length > cap)
|
|
153
|
+
if (s && v.length > cap)
|
|
126
154
|
s.capped++;
|
|
127
155
|
return v.slice(0, cap).map((x) => cutValue(x, clip, cap, s));
|
|
128
156
|
}
|
|
@@ -130,18 +158,31 @@ function cutValue(v, clip, cap, s) {
|
|
|
130
158
|
return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, cutValue(x, clip, cap, s)]));
|
|
131
159
|
return v;
|
|
132
160
|
}
|
|
161
|
+
// A RowKeep as the fields it names, each with its subfields (null: kept whole), built once per
|
|
162
|
+
// RowKeep (3.0.2: through 3.0.1, once per row cut; a 2,000-row answer's structured cut built it
|
|
163
|
+
// some 12,000 times). A RowKeep is never changed once built.
|
|
164
|
+
const wantedOf = (() => {
|
|
165
|
+
const built = new WeakMap();
|
|
166
|
+
return (keep) => {
|
|
167
|
+
let wanted = built.get(keep);
|
|
168
|
+
if (!wanted)
|
|
169
|
+
built.set(keep, (wanted = new Map(keep.map((k) => (typeof k === "string" ? [k, null] : [k[0], k[1]])))));
|
|
170
|
+
return wanted;
|
|
171
|
+
};
|
|
172
|
+
})();
|
|
133
173
|
function cutRow(row, keep, level, s, prefix = "") {
|
|
134
|
-
const wanted =
|
|
174
|
+
const wanted = wantedOf(keep);
|
|
135
175
|
const cap = level.cap ?? Number.POSITIVE_INFINITY;
|
|
136
176
|
const out = {};
|
|
137
|
-
for (const
|
|
177
|
+
for (const k of Object.keys(row)) {
|
|
178
|
+
const v = row[k];
|
|
138
179
|
if (!wanted.has(k)) {
|
|
139
|
-
s
|
|
180
|
+
s?.left.add(`${prefix}${k}`);
|
|
140
181
|
continue;
|
|
141
182
|
}
|
|
142
183
|
const sub = wanted.get(k);
|
|
143
184
|
if (sub && Array.isArray(v)) {
|
|
144
|
-
if (v.length > cap)
|
|
185
|
+
if (s && v.length > cap)
|
|
145
186
|
s.capped++;
|
|
146
187
|
out[k] = v.slice(0, cap).map((e) => (isPlainObject(e) ? cutRow(e, sub, level, s, `${prefix}${k}[].`) : cutValue(e, level.clip, cap, s)));
|
|
147
188
|
}
|
|
@@ -222,6 +263,7 @@ export const FIRST_TEXT_BLOCK = {
|
|
|
222
263
|
export const FIRST_TEXT_BLOCK_BESIDE_A_CUT = {
|
|
223
264
|
opens: (chars) => `TEXT CUT: the API's answer is ${chars} characters of JSON, more than the ${DATA_TEXT_BUDGET} the first text block holds, so that block is cut, and structuredContent.data carries the answer whole with full: true (DATA CUT, below).`,
|
|
224
265
|
leftOut: (n, which) => `the other ${n} are left out of it${which}, and structuredContent.data holds them with full: true`,
|
|
266
|
+
full: true,
|
|
225
267
|
};
|
|
226
268
|
/** The first text block for `data` within DATA_TEXT_BUDGET, and the TEXT CUT sentence when it is cut. */
|
|
227
269
|
export function dataText(data, cut, place = FIRST_TEXT_BLOCK) {
|
|
@@ -241,59 +283,271 @@ export function dataText(data, cut, place = FIRST_TEXT_BLOCK) {
|
|
|
241
283
|
if (cut && isPlainObject(data) && Array.isArray(rows) && rows.length > 0 && cut.levels.length > 0) {
|
|
242
284
|
const n = rows.length;
|
|
243
285
|
// The lists beside the rows, cut once; tried with the first level after it fails with them
|
|
244
|
-
// whole, and with every later level.
|
|
245
|
-
const
|
|
286
|
+
// whole, and with every later level. (Those named beforeRows, #3272, are cut only below.)
|
|
287
|
+
const early = (cut.sides ?? []).filter((side) => !side.beforeRows);
|
|
288
|
+
const sides = early.length ? cutSides(data, early) : { fields: {}, said: [] };
|
|
246
289
|
const sided = sides.said.length > 0;
|
|
247
290
|
const attempts = cut.levels.flatMap((level, i) => (i === 0 && sided ? [{ level, sided: false }, { level, sided: true }] : [{ level, sided }]));
|
|
291
|
+
// The order rows leave the text in: those the tool names first (by value, in its order, each
|
|
292
|
+
// from the end), then the rest from the end.
|
|
293
|
+
const firstOut = cut.leaveOutFirst;
|
|
294
|
+
const isNamed = (i) => firstOut !== undefined && firstOut.values.includes(String(field(rows[i], firstOut.field)));
|
|
295
|
+
const lastOut = cut.leaveOutLast;
|
|
296
|
+
const isLast = (i) => lastOut !== undefined && (lastOut.fields.some((f) => field(rows[i], f) === true) || lastOut.has?.(rows[i]) === true);
|
|
297
|
+
// What a row named last has, as the note says it: "kev_listed or ransomware true" (#2899), or
|
|
298
|
+
// the tool's own words (#3284: "a KEV-listed or ransomware match").
|
|
299
|
+
const lastHas = lastOut ? (lastOut.said ?? `${orListed(lastOut.fields)} true`) : "";
|
|
300
|
+
const lastWhose = lastOut ? (lastOut.said ? `with ${lastOut.said}` : `whose ${orListed(lastOut.fields)} is true`) : "";
|
|
301
|
+
const lastWithout = lastOut ? (lastOut.said ? `without ${lastOut.said}` : `whose ${orListed(lastOut.fields)} is not true`) : "";
|
|
302
|
+
const order = [];
|
|
303
|
+
for (const value of firstOut?.values ?? []) {
|
|
304
|
+
for (let i = n - 1; i >= 0; i--)
|
|
305
|
+
if (field(rows[i], firstOut?.field ?? "") === value)
|
|
306
|
+
order.push(i);
|
|
307
|
+
}
|
|
308
|
+
const named = order.length;
|
|
309
|
+
const queued = new Set(order);
|
|
310
|
+
for (let i = n - 1; i >= 0; i--)
|
|
311
|
+
if (!queued.has(i) && !isLast(i))
|
|
312
|
+
order.push(i);
|
|
313
|
+
for (let i = n - 1; i >= 0; i--)
|
|
314
|
+
if (!queued.has(i) && isLast(i))
|
|
315
|
+
order.push(i);
|
|
316
|
+
// Rows leave `cutRows` in that order, the first `upTo` of it at most, until the text fits; the
|
|
317
|
+
// answer so cut, its sentences after those `said` gives for the number of named rows that left,
|
|
318
|
+
// or undefined where it does not fit.
|
|
319
|
+
const leaving = (cutRows, beside, said, upTo = n) => {
|
|
320
|
+
const lens = cutRows.map((r) => 4 + JSON.stringify(r).length);
|
|
321
|
+
const emptyLen = lined({ ...beside, [cut.rows]: [] }).length;
|
|
322
|
+
const textLen = (kept, sum) => (kept === 0 ? emptyLen : emptyLen + 2 + sum + 2 * kept);
|
|
323
|
+
const out = new Set();
|
|
324
|
+
let sum = lens.reduce((t, x) => t + x, 0);
|
|
325
|
+
for (const i of order.slice(0, upTo)) {
|
|
326
|
+
if (textLen(n - out.size, sum) <= DATA_TEXT_BUDGET)
|
|
327
|
+
break;
|
|
328
|
+
out.add(i);
|
|
329
|
+
sum -= lens[i];
|
|
330
|
+
}
|
|
331
|
+
const kept = cutRows.filter((_, i) => !out.has(i));
|
|
332
|
+
const shown = lined({ ...beside, [cut.rows]: kept });
|
|
333
|
+
if (kept.length === 0 || shown.length > DATA_TEXT_BUDGET)
|
|
334
|
+
return undefined;
|
|
335
|
+
const outFirst = [...out].filter(isNamed).length;
|
|
336
|
+
// #2899: the rows left out are the last in order unless a row named last stayed behind them.
|
|
337
|
+
const inOrder = [...out].every((i) => i >= n - out.size);
|
|
338
|
+
// #3284: the same of the rows left out that the tool does not name first: the last of the
|
|
339
|
+
// rest unless a row named last stayed behind them.
|
|
340
|
+
const restOut = [...out].filter((i) => !isNamed(i));
|
|
341
|
+
const restIdx = [...Array(n).keys()].filter((i) => !isNamed(i));
|
|
342
|
+
const restInOrder = restOut.every((i) => restIdx.indexOf(i) >= restIdx.length - restOut.length);
|
|
343
|
+
const butLast = lastOut ? ` but for any ${lastWhose}${lastOut.why ? ` (${lastOut.why})` : ""}, which leave it last` : "";
|
|
344
|
+
// #3284: the rest, where a row named last stayed behind them: the last of those without one,
|
|
345
|
+
// or, where rows named last left too, each without one and then the last of those with one
|
|
346
|
+
// (said short: the note counts against the structured result's budget).
|
|
347
|
+
const lastOutN = [...out].filter(isLast).length;
|
|
348
|
+
const restOf = (m) => restInOrder || !lastOut
|
|
349
|
+
? `, and the last ${m} of the rest`
|
|
350
|
+
: lastOutN === 0
|
|
351
|
+
? `, and the last ${m} of the rest ${lastWithout}`
|
|
352
|
+
: `, and ${m} of the rest: each ${lastWithout}, then the last ${lastOutN} ${lastWhose}`;
|
|
353
|
+
const which = outFirst > 0
|
|
354
|
+
? `: the ${outFirst} whose ${firstOut?.field} is ${(firstOut?.values ?? []).join(" or ")} (${firstOut?.why})${out.size > outFirst ? restOf(out.size - outFirst) : ""}`
|
|
355
|
+
: inOrder || !lastOut
|
|
356
|
+
? `, the last ${out.size} in order`
|
|
357
|
+
: !lastOut.said
|
|
358
|
+
? `, the last ${out.size} in order${butLast}`
|
|
359
|
+
: lastOutN === 0
|
|
360
|
+
? `, the last ${out.size} ${lastWithout}`
|
|
361
|
+
: `: each ${lastWithout}, then the last ${lastOutN} ${lastWhose}`;
|
|
362
|
+
const elsewhere = place.leftOut ? place.leftOut(out.size, which) : `the other ${out.size} are in structuredContent.data only${which}`;
|
|
363
|
+
const left = out.size > 0 ? `Only ${kept.length} of the ${n} rows of ${cut.rows} are in it; ${elsewhere}.` : "";
|
|
364
|
+
// #2899: each row named last that the text still leaves out, by name.
|
|
365
|
+
const lastLeft = [...out].filter(isLast).sort((a, b) => a - b);
|
|
366
|
+
const named_ = () => {
|
|
367
|
+
if (!lastOut?.runs)
|
|
368
|
+
return listed(lastLeft.map((i) => String(field(rows[i], lastOut?.name ?? "") ?? `${cut.rows}[${i}]`)));
|
|
369
|
+
// #3284: by their numbers, in runs, at most OTHERS_LEFT_LISTED parts.
|
|
370
|
+
const ids = lastLeft.map((i) => field(rows[i], lastOut.name)).map((v, j) => (typeof v === "number" ? v : lastLeft[j])).sort((a, b) => a - b);
|
|
371
|
+
const r = runsSaid(ids);
|
|
372
|
+
return `at ${lastOut.name} ${r.said}${r.more > 0 ? `, and ${r.more} more past ${lastOut.name} ${r.last}` : ""}`;
|
|
373
|
+
};
|
|
374
|
+
const lastSaid = lastOut && lastLeft.length ? `Of the rows left out of it, ${lastLeft.length} ${lastLeft.length === 1 ? "has" : "have"} ${lastHas}: ${named_()}.` : "";
|
|
375
|
+
return { text: shown, said: say(...said(outFirst, out), left, lastSaid, cut.whole, out.size > 0 ? cut.page?.(kept.length, n, data) : undefined) };
|
|
376
|
+
};
|
|
248
377
|
let cutRows = [];
|
|
249
378
|
let beside = data;
|
|
250
379
|
for (const attempt of attempts) {
|
|
251
380
|
const { level } = attempt;
|
|
381
|
+
beside = attempt.sided ? { ...data, ...sides.fields } : data;
|
|
382
|
+
sidesSaid = attempt.sided ? sides.said : [];
|
|
383
|
+
// #3269: before any row is cut to the level outFirstBefore names, the rows leaveOutFirst
|
|
384
|
+
// names are cut to the leanest level, the others kept at the level before, and then leave,
|
|
385
|
+
// as many as must. Where even that does not fit, the cut goes on as it did through 3.0.2:
|
|
386
|
+
// every row at the next level, then rows leaving, those named first.
|
|
387
|
+
const li = cut.levels.indexOf(level);
|
|
388
|
+
if (cut.outFirstBefore !== undefined && li === cut.outFirstBefore && li > 0 && named > 0 && attempt === attempts.find((a) => a.level === level)) {
|
|
389
|
+
const prev = cut.levels[li - 1];
|
|
390
|
+
const lean = cut.levels[cut.levels.length - 1];
|
|
391
|
+
const s = { left: new Set(), clipped: 0, capped: 0 };
|
|
392
|
+
const sn = { left: new Set(), clipped: 0, capped: 0 };
|
|
393
|
+
// #3269's review: the named rows whose strings the lean level clips, or whose lists it
|
|
394
|
+
// caps, by index, so the sentence says so of the ones kept (round 1 said only the fields
|
|
395
|
+
// they leave out, and a clean row's 150-character purl came back clipped unsaid).
|
|
396
|
+
const clippedAt = new Set();
|
|
397
|
+
const cappedAt = new Set();
|
|
398
|
+
const apart = rows.map((r, i) => {
|
|
399
|
+
if (!isPlainObject(r))
|
|
400
|
+
return cutValue(r, prev.clip, prev.cap ?? Number.POSITIVE_INFINITY, s);
|
|
401
|
+
if (!isNamed(i))
|
|
402
|
+
return cutRow(r, prev.keep, prev, s);
|
|
403
|
+
const [clipped, capped] = [sn.clipped, sn.capped];
|
|
404
|
+
const lean_ = cutRow(r, lean.keep, lean, sn);
|
|
405
|
+
if (sn.clipped > clipped)
|
|
406
|
+
clippedAt.add(i);
|
|
407
|
+
if (sn.capped > capped)
|
|
408
|
+
cappedAt.add(i);
|
|
409
|
+
return lean_;
|
|
410
|
+
});
|
|
411
|
+
const extra = [...sn.left].filter((f) => !s.left.has(f));
|
|
412
|
+
const apartSaid = (left, out) => {
|
|
413
|
+
const parts = [];
|
|
414
|
+
if (extra.length)
|
|
415
|
+
parts.push(`leave out ${listed(extra)} too`);
|
|
416
|
+
if ([...clippedAt].some((i) => !out.has(i)))
|
|
417
|
+
parts.push(`every string in each of them longer than ${lean.clip} characters keeps its first ${lean.clip}, ending in "…"`);
|
|
418
|
+
if (lean.cap !== undefined && [...cappedAt].some((i) => !out.has(i)))
|
|
419
|
+
parts.push(`every list in each of them longer than ${lean.cap} entries keeps its first ${lean.cap}`);
|
|
420
|
+
return named - left > 0 && parts.length ? `The ${named - left} rows of ${cut.rows} in it whose ${firstOut?.field} is ${(firstOut?.values ?? []).join(" or ")} ${parts.join("; ")}.` : "";
|
|
421
|
+
};
|
|
422
|
+
const prevSaid = levelSaid(cut.rows, n, prev, s);
|
|
423
|
+
const shown = lined({ ...beside, [cut.rows]: apart });
|
|
424
|
+
if (shown.length <= DATA_TEXT_BUDGET)
|
|
425
|
+
return { text: shown, said: say(prevSaid, apartSaid(0, new Set()), ...sidesSaid, cut.whole) };
|
|
426
|
+
// Only the named rows leave here.
|
|
427
|
+
const r = leaving(apart, beside, (left, out) => [prevSaid, apartSaid(left, out), ...sidesSaid], named);
|
|
428
|
+
if (r)
|
|
429
|
+
return r;
|
|
430
|
+
}
|
|
252
431
|
const s = { left: new Set(), clipped: 0, capped: 0 };
|
|
253
432
|
const cap = level.cap ?? Number.POSITIVE_INFINITY;
|
|
254
433
|
cutRows = rows.map((r) => (isPlainObject(r) ? cutRow(r, level.keep, level, s) : cutValue(r, level.clip, cap, s)));
|
|
255
434
|
rowsSaid = levelSaid(cut.rows, n, level, s);
|
|
256
|
-
beside = attempt.sided ? { ...data, ...sides.fields } : data;
|
|
257
|
-
sidesSaid = attempt.sided ? sides.said : [];
|
|
258
435
|
const shown = lined({ ...beside, [cut.rows]: cutRows });
|
|
259
436
|
if (shown.length <= DATA_TEXT_BUDGET)
|
|
260
437
|
return { text: shown, said: say(rowsSaid, ...sidesSaid, cut.whole) };
|
|
261
438
|
}
|
|
262
|
-
// No level fits:
|
|
263
|
-
//
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
const
|
|
268
|
-
const
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
439
|
+
// No level fits. #3272: the lists the tool names beforeRows, whole until now, are cut as each
|
|
440
|
+
// says, one after another in the order named, until the text fits with every row at the
|
|
441
|
+
// leanest level; then they lose entries, from the end, the last named first, each emptied
|
|
442
|
+
// before the next loses any (a DataCut's keepAll steps, in the same order); the note says
|
|
443
|
+
// what each keeps.
|
|
444
|
+
const spent = (cut.sides ?? []).filter((side) => side.beforeRows);
|
|
445
|
+
for (const side of spent) {
|
|
446
|
+
const one = cutSides(data, [side]);
|
|
447
|
+
if (!one.said.length)
|
|
448
|
+
continue;
|
|
449
|
+
beside = { ...beside, ...one.fields };
|
|
450
|
+
sidesSaid = [...sidesSaid, ...one.said];
|
|
451
|
+
const shown = lined({ ...beside, [cut.rows]: cutRows });
|
|
452
|
+
if (shown.length <= DATA_TEXT_BUDGET)
|
|
453
|
+
return { text: shown, said: say(rowsSaid, ...sidesSaid, cut.whole) };
|
|
273
454
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
const
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
455
|
+
// 3.1.0's phase A review: the lists named with `capped` have each entry's own list cut to its
|
|
456
|
+
// first c entries, the most that fit, before any entry of a list leaves (#3272's text emptied
|
|
457
|
+
// fixed_branches once one package of some 400 ranges did not fit alone, while 3.0.2's every-list
|
|
458
|
+
// cap kept 32 packages; data caps each package's ranges instead, and so does the text now).
|
|
459
|
+
const capping = spent.filter((side) => side.capped && Array.isArray(beside[side.list]) && beside[side.list].length > 0);
|
|
460
|
+
const caps = new Map();
|
|
461
|
+
const capList = (side, v, c) => v.map((e) => {
|
|
462
|
+
const inner = isPlainObject(e) ? e[side.capped.list] : undefined;
|
|
463
|
+
return isPlainObject(e) && Array.isArray(inner) && inner.length > c ? { ...e, [side.capped.list]: inner.slice(0, c), ...side.capped.mark(e, c) } : e;
|
|
464
|
+
});
|
|
465
|
+
const longest = (v, list) => v.reduce((m, e) => Math.max(m, Array.isArray(field(e, list)) ? field(e, list).length : 0), 0);
|
|
466
|
+
// The most c (from `least` up to the longest list less one) at which the text fits, or undefined.
|
|
467
|
+
const mostCap = (side, whole, with_) => {
|
|
468
|
+
const at = (c) => lined({ ...with_, [side.list]: capList(side, whole, c), [cut.rows]: cutRows }).length <= DATA_TEXT_BUDGET;
|
|
469
|
+
let lo = side.capped.least;
|
|
470
|
+
let hi = longest(whole, side.capped.list) - 1;
|
|
471
|
+
if (hi < lo || !at(lo))
|
|
472
|
+
return undefined;
|
|
473
|
+
while (lo < hi) {
|
|
474
|
+
const mid = Math.ceil((lo + hi) / 2);
|
|
475
|
+
if (at(mid))
|
|
476
|
+
lo = mid;
|
|
477
|
+
else
|
|
478
|
+
hi = mid - 1;
|
|
479
|
+
}
|
|
480
|
+
return lo;
|
|
481
|
+
};
|
|
482
|
+
const uncapped = new Map();
|
|
483
|
+
for (const side of capping) {
|
|
484
|
+
const whole = beside[side.list];
|
|
485
|
+
uncapped.set(side.list, whole);
|
|
486
|
+
const c = mostCap(side, whole, beside) ?? side.capped.least;
|
|
487
|
+
caps.set(side.list, c);
|
|
488
|
+
beside = { ...beside, [side.list]: capList(side, whole, c) };
|
|
489
|
+
}
|
|
490
|
+
// The sentence for each capped list: c, and the entries whose own list it cut, by name.
|
|
491
|
+
const capSaid = () => capping.flatMap((side) => {
|
|
492
|
+
const c = caps.get(side.list);
|
|
493
|
+
const kept = beside[side.list].length;
|
|
494
|
+
const inner = (e) => (Array.isArray(field(e, side.capped.list)) ? field(e, side.capped.list).length : 0);
|
|
495
|
+
const names = uncapped.get(side.list).slice(0, kept).filter((e) => inner(e) > c).map(side.capped.name);
|
|
496
|
+
return names.length ? [side.capped.said(c, names, place.full === true)] : [];
|
|
497
|
+
});
|
|
498
|
+
if (capping.length) {
|
|
499
|
+
const shown = lined({ ...beside, [cut.rows]: cutRows });
|
|
500
|
+
if (shown.length <= DATA_TEXT_BUDGET)
|
|
501
|
+
return { text: shown, said: say(rowsSaid, ...sidesSaid, ...capSaid(), cut.whole) };
|
|
502
|
+
}
|
|
503
|
+
const before = [];
|
|
504
|
+
for (const side of [...spent].reverse()) {
|
|
505
|
+
const v = beside[side.list];
|
|
506
|
+
if (!Array.isArray(v) || v.length === 0)
|
|
507
|
+
continue;
|
|
508
|
+
const total = Array.isArray(data[side.list]) ? data[side.list].length : v.length;
|
|
509
|
+
const at = (k) => lined({ ...beside, [side.list]: v.slice(0, k), [cut.rows]: cutRows }).length;
|
|
510
|
+
// Every entry of it does not fit (the text did not, as it stands): the most that do, or none.
|
|
511
|
+
let k = 0;
|
|
512
|
+
if (at(0) <= DATA_TEXT_BUDGET) {
|
|
513
|
+
let hi = v.length - 1;
|
|
514
|
+
while (k < hi) {
|
|
515
|
+
const mid = Math.ceil((k + hi) / 2);
|
|
516
|
+
if (at(mid) <= DATA_TEXT_BUDGET)
|
|
517
|
+
k = mid;
|
|
518
|
+
else
|
|
519
|
+
hi = mid - 1;
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
beside = { ...beside, [side.list]: v.slice(0, k) };
|
|
523
|
+
before.push(k === 0 ? `${side.list} keeps none of its ${total} entries` : `${side.list} keeps its first ${k} of ${total} entries`);
|
|
524
|
+
if (at(k) <= DATA_TEXT_BUDGET)
|
|
282
525
|
break;
|
|
283
|
-
out.add(i);
|
|
284
|
-
sum -= lens[i];
|
|
285
526
|
}
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
const
|
|
290
|
-
const
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
527
|
+
// The room the entries that left give back goes to the capped lists' c (as the structured cut's
|
|
528
|
+
// giveBack does), the first named first.
|
|
529
|
+
for (const side of capping) {
|
|
530
|
+
const whole = uncapped.get(side.list);
|
|
531
|
+
const kept = beside[side.list].length;
|
|
532
|
+
const c = mostCap(side, whole.slice(0, kept), beside);
|
|
533
|
+
if (c !== undefined && c > caps.get(side.list)) {
|
|
534
|
+
caps.set(side.list, c);
|
|
535
|
+
beside = { ...beside, [side.list]: capList(side, whole.slice(0, kept), c) };
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
if (before.length || capping.length) {
|
|
539
|
+
if (before.length)
|
|
540
|
+
sidesSaid = [...sidesSaid, `In it, ${listed(before)}, before any row of ${cut.rows} is left out.`];
|
|
541
|
+
sidesSaid = [...sidesSaid, ...capSaid()];
|
|
542
|
+
const shown = lined({ ...beside, [cut.rows]: cutRows });
|
|
543
|
+
if (shown.length <= DATA_TEXT_BUDGET)
|
|
544
|
+
return { text: shown, said: say(rowsSaid, ...sidesSaid, cut.whole) };
|
|
296
545
|
}
|
|
546
|
+
// Then rows leave the text, those the tool names first, then the rest from the end, until it
|
|
547
|
+
// fits (#2899: those it names last only once every other row is out).
|
|
548
|
+
const r = leaving(cutRows, beside, () => [rowsSaid, ...sidesSaid]);
|
|
549
|
+
if (r)
|
|
550
|
+
return r;
|
|
297
551
|
base = { ...beside, [cut.rows]: cutRows };
|
|
298
552
|
}
|
|
299
553
|
// Step 4: lists, then strings.
|
|
@@ -336,19 +590,71 @@ export const answeringNow = () => calls.getStore();
|
|
|
336
590
|
// tool and directs no model (test/description-policy.test.mjs reads inputSchema descriptions too).
|
|
337
591
|
export const FULL_ARG_DESCRIPTION = "true: data in the structured result is the answer whole, however long. Without it, a structured result past 30,000 characters of JSON has its data cut (DATA CUT): in an order the note states, the entries of its longest lists keep only some of their fields, and then the lists only some of their entries; data_truncated is true, and truncated gives each list cut with its total, the number returned and whether its entries are thinned. Whole data is for a program; a client that sizes structured results, as Claude Code does against MAX_MCP_OUTPUT_TOKENS, can refuse an answer past its cap.";
|
|
338
592
|
export const FULL_ARG = z.boolean().optional().describe(FULL_ARG_DESCRIPTION);
|
|
339
|
-
|
|
593
|
+
// A level as its fields, and what it does inside each entry (#3132).
|
|
594
|
+
const levelOf = (l) => (Array.isArray(l) ? { keep: l } : l);
|
|
595
|
+
// The cap an entry is cut to at a level: `cap`, or the entry's floor where that is more (#3269's
|
|
596
|
+
// second review: EntryLevel.floor), unless `below` (leaveAt's floor, for the one entry kept).
|
|
597
|
+
const capFor = (lv, e, cap, below) => (lv.floor && !below ? Math.max(cap, lv.floor(e)) : cap);
|
|
598
|
+
const thinned = (e, level, cap = Number.POSITIVE_INFINITY, below) => {
|
|
599
|
+
if (!isPlainObject(e))
|
|
600
|
+
return e;
|
|
601
|
+
const lv = levelOf(level);
|
|
602
|
+
const { keep, capped, mark, pick } = lv;
|
|
603
|
+
const c = capFor(lv, e, cap, below);
|
|
604
|
+
const t = cutRow(e, keep, { keep, clip: Number.POSITIVE_INFINITY });
|
|
605
|
+
const inner = capped ? t[capped] : undefined;
|
|
606
|
+
if (capped && Array.isArray(inner) && inner.length > c) {
|
|
607
|
+
// cutRow keeps every entry of the list, in order, so inner[j] is the thinned e[capped][j].
|
|
608
|
+
const whole = e[capped];
|
|
609
|
+
t[capped] = pick && Array.isArray(whole) && whole.length === inner.length ? pick(whole, c).map((j) => inner[j]) : inner.slice(0, c);
|
|
610
|
+
}
|
|
611
|
+
return mark ? { ...t, ...mark(e, c) } : t;
|
|
612
|
+
};
|
|
613
|
+
const innerOf = (e, capped) => (capped !== undefined && Array.isArray(field(e, capped)) ? field(e, capped).length : 0);
|
|
614
|
+
// The longest list named `capped` inside any entry of a list (#3132): the cap a level starts below.
|
|
615
|
+
const longestInside = (v, capped) => Array.isArray(v) ? v.reduce((m, e) => Math.max(m, Array.isArray(field(e, capped)) ? field(e, capped).length : 0), 0) : 0;
|
|
340
616
|
// An entry JSON cannot write (undefined) is written null in a list.
|
|
341
617
|
const entryChars = (e) => JSON.stringify(e)?.length ?? 4;
|
|
342
618
|
const listOf = (data, at) => at.reduce((o, k) => field(o, k), data);
|
|
343
|
-
|
|
619
|
+
// 3.0.2: a list's entries are measured, thinned, only when the cut first asks how many characters
|
|
620
|
+
// some of them take (chars), each thinned, measured and let go; and thinned to be kept only when
|
|
621
|
+
// the cut asks for them (keep), only those it keeps, each once. Through 3.0.1 every list the cut
|
|
622
|
+
// tried held every entry thinned for as long as the cut ran, and a level that caps a list inside
|
|
623
|
+
// each entry thinned every entry at no cap before it tried any: on a 2,000-row check_sbom answer
|
|
624
|
+
// (test/http-memory.test.mjs's) the structured cut allocated 108 MB, against 18 MB on 3.0.0 and 24
|
|
625
|
+
// MB now (V8's sampling heap profiler, collected objects included). What is measured and kept is
|
|
626
|
+
// the same: each entry thinned as before, its characters summed as before.
|
|
627
|
+
// #3269: with `outLevel`, the entries firstOut names are cut to it instead (DataCutStep.outFirst).
|
|
628
|
+
// #3269's second review: with `last`, the entries not named that it holds leave after those it does
|
|
629
|
+
// not (DataCutStep.leaveLast); with `below`, each entry is cut to `cap` under its floor.
|
|
630
|
+
function listAt(data, path, firstOut, level, newest, cap = Number.POSITIVE_INFINITY, levelAt, outLevel, last, below, ordered, shared) {
|
|
344
631
|
const at = path.split(".");
|
|
345
632
|
const v = listOf(data, at);
|
|
346
633
|
if (!Array.isArray(v) || v.length === 0)
|
|
347
634
|
return undefined;
|
|
348
635
|
const n = v.length;
|
|
349
|
-
const
|
|
350
|
-
const
|
|
351
|
-
|
|
636
|
+
const isNamed = (i) => firstOut !== undefined && firstOut.values.includes(field(v[i], firstOut.field));
|
|
637
|
+
const cutOne = (i) => (outLevel && isNamed(i) ? thinned(v[i], outLevel) : level ? thinned(v[i], level, cap, below) : v[i]);
|
|
638
|
+
const cache = shared ?? { made: [] };
|
|
639
|
+
const made = cache.made;
|
|
640
|
+
const entry = (i) => {
|
|
641
|
+
if (!level && !outLevel)
|
|
642
|
+
return v[i];
|
|
643
|
+
if (!(i in made))
|
|
644
|
+
made[i] = cutOne(i);
|
|
645
|
+
return made[i];
|
|
646
|
+
};
|
|
647
|
+
const thin = level ? levelOf(level).keep : undefined;
|
|
648
|
+
// A capped level's clause only where its cap cuts an entry's list (#3139's review: a package
|
|
649
|
+
// with one range is not "at most its first 1"), each entry at its own floor or more (#3269's
|
|
650
|
+
// second review), and told whether some entry's floor is above the cap.
|
|
651
|
+
const lv = level ? levelOf(level) : undefined;
|
|
652
|
+
const bites = lv?.capped === undefined || v.some((e) => innerOf(e, lv.capped) > capFor(lv, e, cap, below));
|
|
653
|
+
const raised = lv?.capped !== undefined && lv.floor !== undefined && !below && v.some((e) => innerOf(e, lv.capped) > cap && lv.floor(e) > cap);
|
|
654
|
+
const thinSaid = lv && bites ? (below ? (lv.below ?? lv.said)?.(cap) : lv.said?.(cap, raised)) : undefined;
|
|
655
|
+
// The indices in the order they leave: from the end, or, for a series kept newest, from the start;
|
|
656
|
+
// those firstOut names first, value by value; then, with `last`, the others it does not hold,
|
|
657
|
+
// then those it holds.
|
|
352
658
|
const leavingOrder = (i) => (newest ? i : n - 1 - i);
|
|
353
659
|
const order = [];
|
|
354
660
|
for (const value of firstOut?.values ?? [])
|
|
@@ -357,27 +663,73 @@ function listAt(data, path, firstOut, thin, newest) {
|
|
|
357
663
|
order.push(leavingOrder(j));
|
|
358
664
|
const named = order.length;
|
|
359
665
|
const queued = new Set(order);
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
666
|
+
const held = (i) => last !== undefined && last.has(v[i]);
|
|
667
|
+
for (const late of last ? [false, true] : [false])
|
|
668
|
+
for (let j = 0; j < n; j++)
|
|
669
|
+
if (!queued.has(leavingOrder(j)) && (!last || held(leavingOrder(j)) === late))
|
|
670
|
+
order.push(leavingOrder(j));
|
|
671
|
+
// #3269's second review: an order the cut chose (the rows kept by fit, cutData), named first.
|
|
672
|
+
if (ordered && ordered.length === n)
|
|
673
|
+
order.splice(0, n, ...ordered);
|
|
674
|
+
// The characters of the entries in the order they leave, summed: leaving[j] is the first j's.
|
|
675
|
+
let leaving;
|
|
676
|
+
const lensOf = () => {
|
|
677
|
+
if (cache.lens)
|
|
678
|
+
return cache.lens;
|
|
679
|
+
const lens = [];
|
|
680
|
+
for (let i = 0; i < n; i++)
|
|
681
|
+
lens.push(entryChars(i in made ? made[i] : cutOne(i)));
|
|
682
|
+
cache.lens = lens;
|
|
683
|
+
return lens;
|
|
684
|
+
};
|
|
685
|
+
const sums = () => {
|
|
686
|
+
if (leaving)
|
|
687
|
+
return leaving;
|
|
688
|
+
const ls = lensOf();
|
|
689
|
+
leaving = [0];
|
|
690
|
+
for (const i of order)
|
|
691
|
+
leaving.push(leaving[leaving.length - 1] + ls[i]);
|
|
692
|
+
return leaving;
|
|
693
|
+
};
|
|
367
694
|
const kept = (k) => Math.max(0, Math.min(n, k));
|
|
695
|
+
const over = new Map();
|
|
696
|
+
const keptAt = (k) => {
|
|
697
|
+
const out = new Set(order.slice(0, n - kept(k)));
|
|
698
|
+
return Array.from({ length: n }, (_, i) => i).filter((i) => !out.has(i));
|
|
699
|
+
};
|
|
368
700
|
return {
|
|
369
701
|
path,
|
|
370
702
|
at,
|
|
371
703
|
n,
|
|
372
704
|
firstOut,
|
|
373
705
|
thin,
|
|
706
|
+
...(thinSaid ? { thinSaid } : {}),
|
|
374
707
|
...(newest ? { newest } : {}),
|
|
375
708
|
named,
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
709
|
+
...(levelAt !== undefined ? { level: levelAt } : {}),
|
|
710
|
+
cap,
|
|
711
|
+
over,
|
|
712
|
+
...(outLevel && named > 0 ? { outThin: levelOf(outLevel).keep } : {}),
|
|
713
|
+
isNamed,
|
|
714
|
+
...(last ? { leaveLast: last } : {}),
|
|
715
|
+
othersOut: (k) => {
|
|
716
|
+
const out = order.slice(0, n - kept(k)).filter((i) => !isNamed(i));
|
|
717
|
+
const own = (i) => (last ? field(v[i], last.id) : undefined);
|
|
718
|
+
const byPosition = out.some((i) => typeof own(i) !== "number" || !Number.isFinite(own(i)));
|
|
719
|
+
const id = (i) => (byPosition ? i + 1 : own(i));
|
|
720
|
+
const asc = (xs) => xs.sort((a, b) => id(a) - id(b));
|
|
721
|
+
return { held: asc(out.filter(held)), rest: asc(out.filter((i) => !held(i))), id, byPosition };
|
|
722
|
+
},
|
|
723
|
+
...(below ? { below } : {}),
|
|
724
|
+
order,
|
|
725
|
+
charsOf: (i) => lensOf()[i],
|
|
726
|
+
withOrder: (o) => listAt(data, path, firstOut, level, newest, cap, levelAt, outLevel, last, below, o, cache),
|
|
727
|
+
chars: (k) => {
|
|
728
|
+
const l = sums();
|
|
729
|
+
return l[n] - l[n - kept(k)] + Math.max(0, kept(k) - 1);
|
|
380
730
|
},
|
|
731
|
+
kept: keptAt,
|
|
732
|
+
keep: (k) => keptAt(k).map((i) => (over.has(i) ? over.get(i)?.entry : entry(i))),
|
|
381
733
|
};
|
|
382
734
|
}
|
|
383
735
|
// `o` with the list at `at` replaced by `v`, each object on the way copied, its keys in their order.
|
|
@@ -388,14 +740,97 @@ function withList(o, at, v) {
|
|
|
388
740
|
const orListed = (xs) => (xs.length < 2 ? xs.join("") : `${xs.slice(0, -1).join(", ")} or ${xs[xs.length - 1]}`);
|
|
389
741
|
// The fields a thinning level keeps, as a sentence says them.
|
|
390
742
|
const keepSaid = (keep) => listed(keep.map((x) => (typeof x === "string" ? x : `${x[0]} (each with ${listed([...x[1]])})`)));
|
|
743
|
+
// The clause on the entries that took back detail (#3132's review): their positions among those
|
|
744
|
+
// kept, in their order in data, grouped where one level and cap holds a run of them (#3132's second
|
|
745
|
+
// review: not always the first kept, since give-back passes an entry that cannot take any back).
|
|
746
|
+
function givenBackSaid(list, k) {
|
|
747
|
+
if (list.over.size === 0)
|
|
748
|
+
return "";
|
|
749
|
+
const kept = list.kept(k);
|
|
750
|
+
const runs = [];
|
|
751
|
+
kept.forEach((i, pos) => {
|
|
752
|
+
const g = list.over.get(i);
|
|
753
|
+
if (!g)
|
|
754
|
+
return;
|
|
755
|
+
const last = runs[runs.length - 1];
|
|
756
|
+
if (last && last.to === pos && last.g.level === g.level && last.g.cap === g.cap)
|
|
757
|
+
last.to = pos + 1;
|
|
758
|
+
else
|
|
759
|
+
runs.push({ from: pos + 1, to: pos + 1, g });
|
|
760
|
+
});
|
|
761
|
+
if (runs.length === 0)
|
|
762
|
+
return "";
|
|
763
|
+
const n = runs.reduce((t, r) => t + r.to - r.from + 1, 0);
|
|
764
|
+
const one = (r) => `${r.from === r.to ? `entry ${r.from}` : `entries ${r.from} to ${r.to}`} cut to ${keepSaid(r.g.keep)}${r.g.said ? `, ${r.g.said}` : ""}`;
|
|
765
|
+
return `; with the room left, ${n} of the entries kept take back detail (by position among those kept, in their order in data): ${runs.map(one).join("; ")}`;
|
|
766
|
+
}
|
|
767
|
+
// The most ids, or runs of them, the clause on the entries not named that left lists (#3269's
|
|
768
|
+
// second review): a client calls those again a few calls' worth at a time (workload_triage: at
|
|
769
|
+
// most 5 calls), and past this many it says how many more. #3269's third review: those LeaveLast
|
|
770
|
+
// holds and the others share it, the held ones first.
|
|
771
|
+
export const OTHERS_LEFT_LISTED = 100;
|
|
772
|
+
// Integers, ascending, as a sentence names them, runs of three or more as "a to b", at most `max`
|
|
773
|
+
// parts: the parts, how many integers they say (`n`) and leave out (`more`), and the last said.
|
|
774
|
+
const runsSaid = (xs, max = OTHERS_LEFT_LISTED) => {
|
|
775
|
+
const parts = [];
|
|
776
|
+
// xs[0..j) are said.
|
|
777
|
+
let j = 0;
|
|
778
|
+
while (j < xs.length && parts.length < max) {
|
|
779
|
+
let e = j;
|
|
780
|
+
while (e + 1 < xs.length && xs[e + 1] === xs[e] + 1)
|
|
781
|
+
e++;
|
|
782
|
+
if (e - j >= 2) {
|
|
783
|
+
parts.push(`${xs[j]} to ${xs[e]}`);
|
|
784
|
+
j = e + 1;
|
|
785
|
+
}
|
|
786
|
+
else {
|
|
787
|
+
parts.push(String(xs[j]));
|
|
788
|
+
j++;
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
return { said: listed(parts), parts: parts.length, n: j, more: xs.length - j, last: xs[j - 1] };
|
|
792
|
+
};
|
|
793
|
+
// The clause on the `g` entries not named that left, with LeaveLast: how many, the order they are
|
|
794
|
+
// kept in, and which (by its id), those LeaveLast holds first and apart (#3269's third review);
|
|
795
|
+
// and, where LeaveLast.again gives it, the sentence on checking them again: in that order, in calls
|
|
796
|
+
// of the sizes PageAt.againFit gives.
|
|
797
|
+
// With `counted` false the clause leaves out the count (the sentence has just said it). The order
|
|
798
|
+
// they were kept in is said where it shows (some entry kept after one that left, in input order).
|
|
799
|
+
function othersLeftSaid(list, k, g, fit, counted = true) {
|
|
800
|
+
const ll = list.leaveLast;
|
|
801
|
+
const { held, rest, id, byPosition } = list.othersOut(k);
|
|
802
|
+
const by = byPosition ? `position in ${list.path} (from 1)` : ll.id;
|
|
803
|
+
const h = runsSaid(held.map(id));
|
|
804
|
+
const r = runsSaid(rest.map(id), OTHERS_LEFT_LISTED - h.parts);
|
|
805
|
+
const at = (x) => x.n === 0 ? `not named here (the note names at most ${OTHERS_LEFT_LISTED} of them, or runs of them)` : `at ${by} ${x.said}${x.more > 0 ? `, and ${x.more} more past ${by} ${x.last}` : ""}`;
|
|
806
|
+
const which = held.length === 0
|
|
807
|
+
? `, ${g === 1 ? "without" : "none with"} ${ll.said}, ${at(r)}`
|
|
808
|
+
: rest.length === 0
|
|
809
|
+
? `, ${g === 1 ? "with" : "all with"} ${ll.said}, ${at(h)}`
|
|
810
|
+
: `: the ${held.length === 1 ? "one" : held.length} with ${ll.said} ${at(h)}; the other ${rest.length} ${at(r)}`;
|
|
811
|
+
// Where none of them is one LeaveLast holds and each left after every entry kept (the last
|
|
812
|
+
// ones), the order shows nothing to say (the synthetic's 201-purl probe: 43 rows kept with it
|
|
813
|
+
// unsaid, 42 with it said).
|
|
814
|
+
const kept = list.kept(k).filter((i) => !list.isNamed(i));
|
|
815
|
+
const last = held.length === 0 && (kept.length === 0 || rest.every((i) => i > kept[kept.length - 1]));
|
|
816
|
+
const said = `${counted ? `${g} ` : ""}${g === 1 ? ll.rest[0] : ll.rest[1]}${which}${last ? "" : ` (${ll.chosen})`}`;
|
|
817
|
+
if (!ll.again || byPosition)
|
|
818
|
+
return { said, again: "" };
|
|
819
|
+
// The entries the note names, in the order it names them.
|
|
820
|
+
const seq = [...held.slice(0, h.n), ...rest.slice(0, r.n)];
|
|
821
|
+
const { sizes, m, alone } = fit.againFit(list.path, seq);
|
|
822
|
+
const a = runsSaid(alone.map(id), Number.POSITIVE_INFINITY);
|
|
823
|
+
return { said, again: ll.again({ k: seq.length, sizes, m, alone: a.said, aloneCount: alone.length }) };
|
|
824
|
+
}
|
|
391
825
|
// What a cut keeps of each list it has cut so far, as truncated and as the DATA CUT sentences.
|
|
392
|
-
function cutSaid(cut, kept, wholeChars, data, over,
|
|
826
|
+
function cutSaid(cut, kept, wholeChars, data, over, fit) {
|
|
393
827
|
const isThinned = ({ list, k }) => list.thin !== undefined && k > 0;
|
|
394
828
|
const cutOnes = kept.filter((x) => x.k < x.list.n || isThinned(x));
|
|
395
829
|
if (cutOnes.length === 0)
|
|
396
830
|
return { said: "" };
|
|
397
831
|
const truncated = Object.fromEntries(cutOnes.map((x) => [x.list.path, { total: x.list.n, returned: x.k, ...(isThinned(x) ? { thinned: true } : {}) }]));
|
|
398
832
|
const parts = [];
|
|
833
|
+
const again = [];
|
|
399
834
|
const byKept = new Map();
|
|
400
835
|
for (const x of cutOnes) {
|
|
401
836
|
const { list, k } = x;
|
|
@@ -403,9 +838,29 @@ function cutSaid(cut, kept, wholeChars, data, over, wholeFit) {
|
|
|
403
838
|
const first = Math.min(out, list.named);
|
|
404
839
|
const end = list.newest ? "oldest" : "last";
|
|
405
840
|
const which = list.firstOut ? `whose ${list.firstOut.field} is ${orListed(list.firstOut.values)} (${list.firstOut.why})` : "";
|
|
406
|
-
|
|
841
|
+
// #3269's second review: with LeaveLast, the entries not named that left are named by its id,
|
|
842
|
+
// so a client can call them alone again, and the order they leave in is said. #3269's third
|
|
843
|
+
// review: and how many of them, in the order named, a call of them alone keeps every one of.
|
|
844
|
+
const others = list.leaveLast && out > first ? othersLeftSaid(list, k, out - first, fit, first > 0) : undefined;
|
|
845
|
+
const rest = others?.said ?? "";
|
|
846
|
+
if (others?.again)
|
|
847
|
+
again.push(others.again);
|
|
848
|
+
const leftOut = out === 0
|
|
849
|
+
? ""
|
|
850
|
+
: rest
|
|
851
|
+
? first === 0
|
|
852
|
+
? `, the ${out} left out being ${rest}`
|
|
853
|
+
: `, the ${out} left out being the ${first} ${which} and ${rest}`
|
|
854
|
+
: first === 0
|
|
855
|
+
? `, the ${end} ${out} left out`
|
|
856
|
+
: first === out
|
|
857
|
+
? `, the ${out} left out being entries ${which}`
|
|
858
|
+
: `, the ${out} left out being the ${first} ${which} and the ${end} ${out - first} of the rest`;
|
|
407
859
|
if (isThinned(x)) {
|
|
408
|
-
|
|
860
|
+
// #3269: the entries outFirst cut apart, those of them kept.
|
|
861
|
+
const apart = list.outThin ? list.kept(k).filter((i) => list.isNamed(i)).length : 0;
|
|
862
|
+
const apartSaid = apart > 0 && list.firstOut ? `, except the ${apart} kept whose ${list.firstOut.field} is ${orListed(list.firstOut.values)}, each cut to ${keepSaid(list.outThin)}` : "";
|
|
863
|
+
parts.push(`${list.path} keeps ${k === list.n ? `all ${k}` : `${k} of its ${list.n}`} entries, each cut to ${keepSaid(list.thin)}${list.thinSaid ? `, ${list.thinSaid}` : ""}${apartSaid}${leftOut}${givenBackSaid(list, k)}`);
|
|
409
864
|
continue;
|
|
410
865
|
}
|
|
411
866
|
if (!list.firstOut || first === 0 || k === 0) {
|
|
@@ -433,9 +888,10 @@ function cutSaid(cut, kept, wholeChars, data, over, wholeFit) {
|
|
|
433
888
|
return p.k === 0 ? `${paths} keep none of their entries` : `${paths} keep their ${which} ${p.k} entries each`;
|
|
434
889
|
});
|
|
435
890
|
const keeps = Object.fromEntries(cutOnes.filter(isThinned).map((x) => [x.list.path, x.list.thin]));
|
|
436
|
-
const page = cut.page?.(truncated, data, { keeps,
|
|
891
|
+
const page = cut.page?.(truncated, data, { keeps, ...fit }) ?? "";
|
|
437
892
|
const said = [
|
|
438
893
|
`DATA CUT: with data whole, this structured result is ${wholeChars} characters of JSON, more than the ${STRUCTURED_BUDGET} it holds unless the call says full: true, so its data is cut: ${cuts.join("; ")}.`,
|
|
894
|
+
...again,
|
|
439
895
|
"truncated gives the total and the number returned of each list cut, and thinned where its entries are cut to some of their fields.",
|
|
440
896
|
page,
|
|
441
897
|
over ? `Cut so, the structured result is still past ${STRUCTURED_BUDGET} characters: data holds no other list to cut.` : "",
|
|
@@ -456,6 +912,8 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
456
912
|
const steps = [...cut.steps, { lists: Object.keys(data).filter((k) => !named.has(k)) }];
|
|
457
913
|
// Each list cut so far, in the order cut, at the level it was thinned to, how many entries it
|
|
458
914
|
// keeps, and the step that cut it.
|
|
915
|
+
// `gone` (#3269): the list's entries outFirst names left before the others were cut further, and
|
|
916
|
+
// may come back once the cut is settled.
|
|
459
917
|
const kept = new Map();
|
|
460
918
|
const apply = (also = []) => [...kept.values(), ...also].reduce((d, { list, k }) => withList(d, list.at, list.keep(k)), data);
|
|
461
919
|
// PageAt.wholeFit: of the list at `path`, the most first entries, each whole, that the result
|
|
@@ -481,27 +939,378 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
481
939
|
wholeFits.set(path, k);
|
|
482
940
|
return k;
|
|
483
941
|
};
|
|
942
|
+
// PageAt.firstFit (#3269's review): of the list at `path`, which a step thins, the most first
|
|
943
|
+
// entries n such that, for every n' up to n, a call answering only the first n' of them keeps
|
|
944
|
+
// every one of them, as this cut would cut that call: it does where
|
|
945
|
+
// - every entry fits thinned (thin[0]), those leaveOutFirst names cut to outFirst's level; or
|
|
946
|
+
// - every entry fits at the floor (thin[floor.at] at its least, each entry at its own
|
|
947
|
+
// EntryLevel.floor or more: #3269's second review), those named so cut, and either none is
|
|
948
|
+
// named or the entries not named, thinned, are past the budget on their own (so the call
|
|
949
|
+
// cannot keep them uncapped by sending the named ones out, and keeps them at the floor
|
|
950
|
+
// instead).
|
|
951
|
+
// A step without a floor is held to its leaveAt level and cap instead, so a page of that size
|
|
952
|
+
// comes back with that level's detail, as #3132's review asked. Each size is counted as this
|
|
953
|
+
// call's: the entries as they are (the first n of a call are these, at the same index), data
|
|
954
|
+
// beside them with this list and the lists of the steps before it empty (a cut empties those
|
|
955
|
+
// first, and a call of fewer purls sends them all), and this call's envelope and note, with the
|
|
956
|
+
// longest DATA CUT sentence such a call can carry while it keeps every entry. A call of fewer
|
|
957
|
+
// purls has a note no longer than this one's (its counts are no larger, the affected components
|
|
958
|
+
// it names are the first of these), but for its TEXT CUT sentences, which can be cut no further
|
|
959
|
+
// than this call's text but can take another form: FIRST_FIT_SLACK holds the difference.
|
|
960
|
+
const FIRST_FIT_SLACK = 600;
|
|
961
|
+
const measures = new Map();
|
|
962
|
+
const measureOf = (path) => {
|
|
963
|
+
const known = measures.get(path);
|
|
964
|
+
if (known !== undefined)
|
|
965
|
+
return known;
|
|
966
|
+
const s = steps.findIndex((st) => st.lists.includes(path));
|
|
967
|
+
const v = listOf(data, path.split("."));
|
|
968
|
+
const m = s >= 0 && steps[s].thin?.length && Array.isArray(v) && v.length > 0 ? measureFor(steps[s], s, path, v) : null;
|
|
969
|
+
measures.set(path, m);
|
|
970
|
+
return m;
|
|
971
|
+
};
|
|
972
|
+
const measureFor = (step, s, path, v) => {
|
|
973
|
+
const thin = step.thin;
|
|
974
|
+
const first = step.leaveOutFirst;
|
|
975
|
+
const out = first ? step.outFirst : undefined;
|
|
976
|
+
const floor = out?.floor;
|
|
977
|
+
const lean = out?.level;
|
|
978
|
+
const n = v.length;
|
|
979
|
+
const isNamed = (e) => first !== undefined && first.values.includes(field(e, first.field));
|
|
980
|
+
const bare = [path, ...steps.slice(0, s).flatMap((st) => st.lists)].reduce((d, p) => (Array.isArray(listOf(d, p.split("."))) ? withList(d, p.split("."), []) : d), data);
|
|
981
|
+
const base = JSON.stringify(bare).length;
|
|
982
|
+
// The level and cap every entry is held to where they are not all kept thinned.
|
|
983
|
+
const atFloor = floor ? floor.at : step.leaveAt ? step.leaveAt.level : thin.length - 1;
|
|
984
|
+
const floorCap = floor ? (levelOf(thin[atFloor]).least ?? 0) : step.leaveAt ? step.leaveAt.cap : Number.POSITIVE_INFINITY;
|
|
985
|
+
// The envelope, its note and the longest DATA CUT sentence of a call that keeps every entry,
|
|
986
|
+
// at each level it can stop at (a capped level's clause said at its least, where it bites).
|
|
987
|
+
const noPage = { steps: cut.steps };
|
|
988
|
+
const stops = [0, ...(floor && out ? Array.from({ length: floor.at - out.before + 1 }, (_, j) => out.before + j) : [atFloor])];
|
|
989
|
+
const env = Math.max(...stops.map((j) => {
|
|
990
|
+
const lv = levelOf(thin[j]);
|
|
991
|
+
const list = listAt(data, path, first, thin[j], step.keepNewest, lv.capped ? (lv.least ?? 0) : Number.POSITIVE_INFINITY, j, lean, step.leaveLast);
|
|
992
|
+
if (!list)
|
|
993
|
+
return 0;
|
|
994
|
+
const { said } = cutSaid(noPage, [{ list, k: n }], wholeChars, data, false, { wholeFit, firstFit: () => 0, againFit: () => ({ sizes: [], m: 0, alone: [] }) });
|
|
995
|
+
return JSON.stringify(structured(null, { [path]: { total: n, returned: n, thinned: true } }, said)).length - "null".length;
|
|
996
|
+
})) +
|
|
997
|
+
base +
|
|
998
|
+
FIRST_FIT_SLACK;
|
|
999
|
+
const chars = (e, level, cap = Number.POSITIVE_INFINITY) => entryChars(thinned(e, level, cap));
|
|
1000
|
+
const once = (f) => {
|
|
1001
|
+
const got = new Map();
|
|
1002
|
+
return (i) => {
|
|
1003
|
+
let c = got.get(i);
|
|
1004
|
+
if (c === undefined)
|
|
1005
|
+
got.set(i, (c = f(i)));
|
|
1006
|
+
return c;
|
|
1007
|
+
};
|
|
1008
|
+
};
|
|
1009
|
+
const named = (i) => isNamed(v[i]);
|
|
1010
|
+
return {
|
|
1011
|
+
v,
|
|
1012
|
+
step,
|
|
1013
|
+
env,
|
|
1014
|
+
named,
|
|
1015
|
+
thinChars: once((i) => chars(v[i], named(i) && lean !== undefined ? lean : thin[0])),
|
|
1016
|
+
floorChars: once((i) => (named(i) && lean !== undefined ? chars(v[i], lean) : chars(v[i], thin[atFloor], floorCap))),
|
|
1017
|
+
};
|
|
1018
|
+
};
|
|
1019
|
+
const firstFits = new Map();
|
|
1020
|
+
const firstFit = (path) => {
|
|
1021
|
+
const known = firstFits.get(path);
|
|
1022
|
+
if (known !== undefined)
|
|
1023
|
+
return known;
|
|
1024
|
+
const ms = measureOf(path);
|
|
1025
|
+
const fit = ms ? firstFitOf(ms) : 0;
|
|
1026
|
+
firstFits.set(path, fit);
|
|
1027
|
+
return fit;
|
|
1028
|
+
};
|
|
1029
|
+
const firstFitOf = ({ v, step, env, named: isNamed, thinChars, floorChars }) => {
|
|
1030
|
+
const out = step.leaveOutFirst ? step.outFirst : undefined;
|
|
1031
|
+
const floor = out?.floor;
|
|
1032
|
+
const n = v.length;
|
|
1033
|
+
let thinnedAll = 0;
|
|
1034
|
+
let atFloorAll = 0;
|
|
1035
|
+
let others = 0;
|
|
1036
|
+
let othersAlone = 0;
|
|
1037
|
+
let named = 0;
|
|
1038
|
+
for (let i = 0; i < n; i++) {
|
|
1039
|
+
const nm = isNamed(i);
|
|
1040
|
+
if (nm)
|
|
1041
|
+
named++;
|
|
1042
|
+
thinnedAll += thinChars(i);
|
|
1043
|
+
atFloorAll += floorChars(i);
|
|
1044
|
+
if (!nm) {
|
|
1045
|
+
others++;
|
|
1046
|
+
othersAlone += thinChars(i);
|
|
1047
|
+
}
|
|
1048
|
+
const commas = i;
|
|
1049
|
+
const keepsThinned = env + thinnedAll + commas <= STRUCTURED_BUDGET;
|
|
1050
|
+
const keepsAtFloor = env + atFloorAll + commas <= STRUCTURED_BUDGET;
|
|
1051
|
+
// The entries not named, thinned, without the rest of the result: a lower bound.
|
|
1052
|
+
const othersPast = othersAlone + Math.max(0, others - 1) + 2 > STRUCTURED_BUDGET;
|
|
1053
|
+
if (!(keepsThinned || (keepsAtFloor && (named === 0 || !out || (floor !== undefined && othersPast)))))
|
|
1054
|
+
return i;
|
|
1055
|
+
}
|
|
1056
|
+
return n;
|
|
1057
|
+
};
|
|
1058
|
+
// PageAt.againFit (#3269's third review): of the entries `seq` names (none of them one
|
|
1059
|
+
// leaveOutFirst names, so a call of them alone keeps every one where they fit at their floor, as
|
|
1060
|
+
// firstFit counts), those that do not fit at it even alone leave `seq` (`alone`). Of the rest,
|
|
1061
|
+
// each of the first LeaveLast.againCalls calls takes the most of those left, in order, that fit
|
|
1062
|
+
// at the floor beside the result's envelope (this call's, its note with LeaveLast.noteOver's
|
|
1063
|
+
// characters for a call of that many, and a DATA CUT sentence keeping them all); of the entries
|
|
1064
|
+
// after those, m is the most such that every m of them in a row fit so. So each call of those
|
|
1065
|
+
// sizes keeps every one, and so does a call of fewer of them in a row. Counted once per `seq`
|
|
1066
|
+
// (the cut sizes the same note many times).
|
|
1067
|
+
const againFits = new Map();
|
|
1068
|
+
const againFit = (path, seq) => {
|
|
1069
|
+
const key = `${path}:${seq.join(",")}`;
|
|
1070
|
+
const known = againFits.get(key);
|
|
1071
|
+
if (known)
|
|
1072
|
+
return known;
|
|
1073
|
+
const got = againFitOf(path, seq);
|
|
1074
|
+
againFits.set(key, got);
|
|
1075
|
+
return got;
|
|
1076
|
+
};
|
|
1077
|
+
const againFitOf = (path, seq) => {
|
|
1078
|
+
const ms = measureOf(path);
|
|
1079
|
+
if (!ms || seq.length === 0)
|
|
1080
|
+
return { sizes: [], m: 0, alone: [] };
|
|
1081
|
+
const ll = ms.step.leaveLast;
|
|
1082
|
+
const over = ll?.noteOver?.(seq.map((i) => ms.v[i]), ms.v) ?? (() => 0);
|
|
1083
|
+
const room = (m) => STRUCTURED_BUDGET - ms.env - over(m);
|
|
1084
|
+
const one = room(1);
|
|
1085
|
+
const alone = seq.filter((i) => ms.floorChars(i) > one);
|
|
1086
|
+
const rest = alone.length ? seq.filter((i) => ms.floorChars(i) <= one) : seq;
|
|
1087
|
+
const calls = ll?.againCalls ?? 0;
|
|
1088
|
+
// sums[j]: the characters of rest's first j entries; rest[a..b) fits as one call.
|
|
1089
|
+
const sums = [0];
|
|
1090
|
+
for (const i of rest)
|
|
1091
|
+
sums.push(sums[sums.length - 1] + ms.floorChars(i));
|
|
1092
|
+
const fitsAt = (a, b) => sums[b] - sums[a] + (b - a - 1) <= room(b - a);
|
|
1093
|
+
const sizes = [];
|
|
1094
|
+
let at = 0;
|
|
1095
|
+
while (sizes.length < calls && at < rest.length) {
|
|
1096
|
+
let b = at + 1;
|
|
1097
|
+
while (b < rest.length && fitsAt(at, b + 1))
|
|
1098
|
+
b++;
|
|
1099
|
+
sizes.push(b - at);
|
|
1100
|
+
at = b;
|
|
1101
|
+
}
|
|
1102
|
+
// Every m of rest[at..] in a row fits (fewer, at its end, are within the last m).
|
|
1103
|
+
const fits = (m) => {
|
|
1104
|
+
for (let a = at; a + m <= rest.length; a++)
|
|
1105
|
+
if (!fitsAt(a, a + m))
|
|
1106
|
+
return false;
|
|
1107
|
+
return true;
|
|
1108
|
+
};
|
|
1109
|
+
let m = rest.length > at ? 1 : 0;
|
|
1110
|
+
let hi = rest.length - at;
|
|
1111
|
+
while (m < hi) {
|
|
1112
|
+
const mid = Math.ceil((m + hi) / 2);
|
|
1113
|
+
if (fits(mid))
|
|
1114
|
+
m = mid;
|
|
1115
|
+
else
|
|
1116
|
+
hi = mid - 1;
|
|
1117
|
+
}
|
|
1118
|
+
return { sizes, m, alone };
|
|
1119
|
+
};
|
|
1120
|
+
const fit = { wholeFit, firstFit, againFit };
|
|
484
1121
|
// The result's size with this step's lists at k entries each: the rest of the result (data
|
|
485
1122
|
// null), data with this step's lists empty (`base`), and their entries.
|
|
486
1123
|
const size = (lists, k, base) => {
|
|
487
1124
|
const all = [...kept.values(), ...lists.map((list) => ({ list, k: Math.min(k, list.n) }))];
|
|
488
|
-
const { truncated, said } = cutSaid(cut, all, wholeChars, data, false,
|
|
1125
|
+
const { truncated, said } = cutSaid(cut, all, wholeChars, data, false, fit);
|
|
489
1126
|
return JSON.stringify(structured(null, truncated, said)).length - "null".length + base + lists.reduce((t, l) => t + l.chars(k), 0);
|
|
490
1127
|
};
|
|
1128
|
+
// #3269's second review: whether an entry of a step's lists that leaveOutFirst does not name holds
|
|
1129
|
+
// any entry of the list leaveAt's level caps (DataCutStep.leaveAt's floor).
|
|
1130
|
+
const holdsInner = (step) => {
|
|
1131
|
+
const inner = step.thin && step.leaveAt ? levelOf(step.thin[step.leaveAt.level]).capped : undefined;
|
|
1132
|
+
if (inner === undefined)
|
|
1133
|
+
return false;
|
|
1134
|
+
const first = step.leaveOutFirst;
|
|
1135
|
+
const named = (e) => first !== undefined && first.values.includes(field(e, first.field));
|
|
1136
|
+
return step.lists.some((p) => {
|
|
1137
|
+
const v = listOf(data, p.split("."));
|
|
1138
|
+
return Array.isArray(v) && v.some((e) => !named(e) && innerOf(e, inner) > 0);
|
|
1139
|
+
});
|
|
1140
|
+
};
|
|
1141
|
+
// #3269's second review: with leaveAt's floor and LeaveLast, the entries kept are chosen in the
|
|
1142
|
+
// reverse of the order they leave in (LeaveLast's first, then the rest, each from the start),
|
|
1143
|
+
// each one kept that fits beside those chosen before it, and the rest leave first: the list so
|
|
1144
|
+
// reordered, and how many it keeps. In the plain order a row too large to fit beside the rows kept
|
|
1145
|
+
// sent every row before it in that order out with it, and left their room empty: a row of 200
|
|
1146
|
+
// KEV-listed matches after 150 purls of an image (40 affected) kept 1 row of 151, in 7,572
|
|
1147
|
+
// characters. The choice is made on the room the result leaves with none kept, then checked as
|
|
1148
|
+
// the result is (its note names the entries left out, by index), the last chosen leaving while it
|
|
1149
|
+
// does not fit, and topped up, each entry not chosen that fits now, in the same order.
|
|
1150
|
+
const keptByFit = (l, base) => {
|
|
1151
|
+
if (size([l], 0, base) > STRUCTURED_BUDGET)
|
|
1152
|
+
return undefined;
|
|
1153
|
+
const named = l.order.filter((i) => l.isNamed(i));
|
|
1154
|
+
const pri = [...l.order].reverse().filter((i) => !l.isNamed(i));
|
|
1155
|
+
const orderOf = (chosen) => [...named, ...l.order.filter((i) => !l.isNamed(i) && !chosen.has(i)), ...l.order.filter((i) => chosen.has(i))];
|
|
1156
|
+
const chosen = new Set();
|
|
1157
|
+
let room = STRUCTURED_BUDGET - size([l], 0, base);
|
|
1158
|
+
for (const i of pri) {
|
|
1159
|
+
const c = l.charsOf(i) + (chosen.size > 0 ? 1 : 0);
|
|
1160
|
+
if (c > room)
|
|
1161
|
+
continue;
|
|
1162
|
+
chosen.add(i);
|
|
1163
|
+
room -= c;
|
|
1164
|
+
}
|
|
1165
|
+
let cur = l.withOrder(orderOf(chosen));
|
|
1166
|
+
let k = chosen.size;
|
|
1167
|
+
while (k > 0 && size([cur], k, base) > STRUCTURED_BUDGET)
|
|
1168
|
+
k--;
|
|
1169
|
+
for (let tries = 0; tries < 64; tries++) {
|
|
1170
|
+
const kept = new Set(cur.kept(k));
|
|
1171
|
+
const left = STRUCTURED_BUDGET - size([cur], k, base);
|
|
1172
|
+
const next = pri.find((i) => !kept.has(i) && cur.charsOf(i) + 1 <= left && size([cur.withOrder(orderOf(new Set([...kept, i])))], k + 1, base) <= STRUCTURED_BUDGET);
|
|
1173
|
+
if (next === undefined)
|
|
1174
|
+
break;
|
|
1175
|
+
cur = cur.withOrder(orderOf(new Set([...kept, next])));
|
|
1176
|
+
k++;
|
|
1177
|
+
}
|
|
1178
|
+
return { list: cur, k };
|
|
1179
|
+
};
|
|
491
1180
|
let fits = false;
|
|
492
1181
|
for (const [s, step] of steps.entries()) {
|
|
493
1182
|
// Every entry whole, then each thinning level in turn, every entry kept; at the leanest level
|
|
494
|
-
// the entries leave until the result fits.
|
|
1183
|
+
// the entries leave until the result fits (with keepAll, #3139, not in this step: see below).
|
|
495
1184
|
const levels = [undefined, ...(step.thin ?? [])];
|
|
496
1185
|
let lists = [];
|
|
497
1186
|
let k = 0;
|
|
1187
|
+
// #3269: once the entries outFirst names have left (gone), each later level cuts them to its
|
|
1188
|
+
// level, so that they can come back so cut, and keeps at most the others (they leave first).
|
|
1189
|
+
const out = step.leaveOutFirst ? step.outFirst : undefined;
|
|
1190
|
+
let gone = false;
|
|
1191
|
+
const at = (level, cap, levelAt, outLevel = gone ? out?.level : undefined, below) => step.lists
|
|
1192
|
+
.map((p) => listAt(data, p, step.leaveOutFirst, level, step.keepNewest, cap, levelAt, outLevel, step.leaveLast, below))
|
|
1193
|
+
.filter((l) => l !== undefined && !kept.has(l.path));
|
|
1194
|
+
// #3269's second review: with leaveAt's floor, the last level tried with every entry kept is
|
|
1195
|
+
// leaveAt's while an entry not named holds inner entries (DataCutStep.leaveAt): no later level
|
|
1196
|
+
// keeps any of them, so the entries leave there, whole, instead.
|
|
1197
|
+
const end = step.leaveAt?.floor && step.thin && holdsInner(step) ? step.leaveAt.level + 1 : levels.length - 1;
|
|
498
1198
|
for (const [i, thin] of levels.entries()) {
|
|
499
|
-
|
|
1199
|
+
if (i > end)
|
|
1200
|
+
break;
|
|
1201
|
+
// #3269: before any entry is cut to thin[out.before], the entries leaveOutFirst names are cut
|
|
1202
|
+
// to out.level, the others kept at the level before; then they leave, as many as must.
|
|
1203
|
+
if (out && !gone && i === out.before + 1) {
|
|
1204
|
+
const apart = at(levels[i - 1], undefined, i - 2, out.level);
|
|
1205
|
+
if (apart.some((l) => l.named > 0)) {
|
|
1206
|
+
const base = JSON.stringify(apply(apart.map((list) => ({ list, k: 0 })))).length;
|
|
1207
|
+
const rest = Math.max(...apart.map((l) => l.n - l.named));
|
|
1208
|
+
const all = Math.max(...apart.map((l) => l.n));
|
|
1209
|
+
// The most entries each list keeps, `lo` at least, cut as `ls` is.
|
|
1210
|
+
const most = (ls, lo) => {
|
|
1211
|
+
let kk = lo;
|
|
1212
|
+
let hi = all;
|
|
1213
|
+
while (kk < hi) {
|
|
1214
|
+
const mid = Math.ceil((kk + hi) / 2);
|
|
1215
|
+
if (size(ls, mid, base) <= STRUCTURED_BUDGET)
|
|
1216
|
+
kk = mid;
|
|
1217
|
+
else
|
|
1218
|
+
hi = mid - 1;
|
|
1219
|
+
}
|
|
1220
|
+
return kk;
|
|
1221
|
+
};
|
|
1222
|
+
if (size(apart, rest, base) <= STRUCTURED_BUDGET) {
|
|
1223
|
+
k = most(apart, rest);
|
|
1224
|
+
lists = apart;
|
|
1225
|
+
fits = true;
|
|
1226
|
+
break;
|
|
1227
|
+
}
|
|
1228
|
+
// #3269's review: the others are capped anyway, so the named entries stay while the
|
|
1229
|
+
// others fit at the floor (OutFirst.floor). #3269's second review: each capped level
|
|
1230
|
+
// holds each entry's own floor (EntryLevel.floor), so the cap starts at its least.
|
|
1231
|
+
const floor = out.floor;
|
|
1232
|
+
if (floor) {
|
|
1233
|
+
const last = floor.at + 1;
|
|
1234
|
+
// With kk entries kept, the richest capped level from thin[before] to thin[floor.at]
|
|
1235
|
+
// that fits at the floor or more, at the most inner entries that fit; none where none.
|
|
1236
|
+
const capped = (kk) => {
|
|
1237
|
+
for (let j = i; j <= last; j++) {
|
|
1238
|
+
const lv = levelOf(levels[j]);
|
|
1239
|
+
if (lv.capped === undefined)
|
|
1240
|
+
continue;
|
|
1241
|
+
const ls = (c) => at(levels[j], c, j - 1, out.level);
|
|
1242
|
+
let m = lv.least ?? 0;
|
|
1243
|
+
if (size(ls(m), kk, base) > STRUCTURED_BUDGET)
|
|
1244
|
+
continue;
|
|
1245
|
+
let hi = Math.max(m, ...step.lists.map((p) => longestInside(listOf(data, p.split(".")), lv.capped)));
|
|
1246
|
+
while (m < hi) {
|
|
1247
|
+
const mid = Math.ceil((m + hi) / 2);
|
|
1248
|
+
if (size(ls(mid), kk, base) <= STRUCTURED_BUDGET)
|
|
1249
|
+
m = mid;
|
|
1250
|
+
else
|
|
1251
|
+
hi = mid - 1;
|
|
1252
|
+
}
|
|
1253
|
+
return ls(m);
|
|
1254
|
+
}
|
|
1255
|
+
return undefined;
|
|
1256
|
+
};
|
|
1257
|
+
let kk = all;
|
|
1258
|
+
let got = capped(all);
|
|
1259
|
+
if (!got) {
|
|
1260
|
+
// Even the floor does not fit with every named entry: they leave, as many as must
|
|
1261
|
+
// for it to, and the others then take the most that fits.
|
|
1262
|
+
const lv = levelOf(levels[last]);
|
|
1263
|
+
const atLeast = at(levels[last], lv.least ?? 0, last - 1, out.level);
|
|
1264
|
+
if (size(atLeast, rest, base) <= STRUCTURED_BUDGET) {
|
|
1265
|
+
kk = most(atLeast, rest);
|
|
1266
|
+
got = capped(kk);
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
if (got) {
|
|
1270
|
+
lists = got;
|
|
1271
|
+
k = kk;
|
|
1272
|
+
if (kk < all)
|
|
1273
|
+
gone = true;
|
|
1274
|
+
fits = true;
|
|
1275
|
+
break;
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
gone = true;
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
lists = at(thin, undefined, i - 1);
|
|
500
1282
|
if (lists.length === 0)
|
|
501
1283
|
break;
|
|
502
1284
|
const base = JSON.stringify(apply(lists.map((list) => ({ list, k: 0 })))).length;
|
|
503
|
-
const top = Math.max(...lists.map((l) => l.n));
|
|
504
|
-
|
|
1285
|
+
const top = Math.max(...lists.map((l) => (gone ? l.n - l.named : l.n)));
|
|
1286
|
+
// #3132: a level that caps a list inside each entry, every entry kept: the most entries of
|
|
1287
|
+
// that inner list, the same number in each, that fit; a level where even none fits is passed.
|
|
1288
|
+
// #3132's review: a level with `least` is taken at that cap or more, or passed.
|
|
1289
|
+
const capped = thin === undefined ? undefined : levelOf(thin).capped;
|
|
1290
|
+
if (capped !== undefined) {
|
|
1291
|
+
const longest = Math.max(0, ...step.lists.map((p) => longestInside(listOf(data, p.split(".")), capped)));
|
|
1292
|
+
const least = (thin === undefined ? undefined : levelOf(thin).least) ?? 0;
|
|
1293
|
+
if (longest > least && size(at(thin, least, i - 1), top, base) <= STRUCTURED_BUDGET) {
|
|
1294
|
+
let m = least;
|
|
1295
|
+
let hi = longest - 1;
|
|
1296
|
+
while (m < hi) {
|
|
1297
|
+
const mid = Math.ceil((m + hi) / 2);
|
|
1298
|
+
if (size(at(thin, mid, i - 1), top, base) <= STRUCTURED_BUDGET)
|
|
1299
|
+
m = mid;
|
|
1300
|
+
else
|
|
1301
|
+
hi = mid - 1;
|
|
1302
|
+
}
|
|
1303
|
+
lists = at(thin, m, i - 1);
|
|
1304
|
+
k = top;
|
|
1305
|
+
fits = true;
|
|
1306
|
+
break;
|
|
1307
|
+
}
|
|
1308
|
+
if (i < end)
|
|
1309
|
+
continue;
|
|
1310
|
+
// The last level, and even its least cap does not fit: the entries go on at that cap.
|
|
1311
|
+
lists = at(thin, least, i - 1);
|
|
1312
|
+
}
|
|
1313
|
+
if (i < end) {
|
|
505
1314
|
if (size(lists, top, base) <= STRUCTURED_BUDGET) {
|
|
506
1315
|
k = top;
|
|
507
1316
|
fits = true;
|
|
@@ -509,7 +1318,32 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
509
1318
|
}
|
|
510
1319
|
continue;
|
|
511
1320
|
}
|
|
512
|
-
if (
|
|
1321
|
+
if (step.keepAll) {
|
|
1322
|
+
// #3139: every entry stays, at this level; the later steps are cut first.
|
|
1323
|
+
k = top;
|
|
1324
|
+
fits = size(lists, top, base) <= STRUCTURED_BUDGET;
|
|
1325
|
+
break;
|
|
1326
|
+
}
|
|
1327
|
+
// #3132's review: entries leave at the step's leaveAt level, where it names one.
|
|
1328
|
+
if (step.leaveAt && step.thin) {
|
|
1329
|
+
const { level, cap } = step.leaveAt;
|
|
1330
|
+
lists = at(step.thin[level], cap, level);
|
|
1331
|
+
}
|
|
1332
|
+
if (step.whole) {
|
|
1333
|
+
// Every entry or none (#3132's review: data.not_sent_purls).
|
|
1334
|
+
if (size(lists, 0, base) <= STRUCTURED_BUDGET) {
|
|
1335
|
+
k = size(lists, top, base) <= STRUCTURED_BUDGET ? top : 0;
|
|
1336
|
+
fits = true;
|
|
1337
|
+
}
|
|
1338
|
+
break;
|
|
1339
|
+
}
|
|
1340
|
+
const chosen = step.leaveAt?.floor && step.leaveLast && lists.length === 1 ? keptByFit(lists[0], base) : undefined;
|
|
1341
|
+
if (chosen) {
|
|
1342
|
+
lists = [chosen.list];
|
|
1343
|
+
k = chosen.k;
|
|
1344
|
+
fits = true;
|
|
1345
|
+
}
|
|
1346
|
+
else if (size(lists, 0, base) <= STRUCTURED_BUDGET) {
|
|
513
1347
|
// The most entries each list of the step keeps with the result inside the budget.
|
|
514
1348
|
let hi = top;
|
|
515
1349
|
while (k < hi) {
|
|
@@ -521,21 +1355,89 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
521
1355
|
}
|
|
522
1356
|
fits = true;
|
|
523
1357
|
}
|
|
1358
|
+
// #3269's second review: not even one entry fits at its floor (one extreme row, its KEV-listed
|
|
1359
|
+
// matches alone past the budget). The one the leaving order keeps last stays, its capped list
|
|
1360
|
+
// at the most that fit below its floor, at least one; where not even one fits, at the first
|
|
1361
|
+
// later level that fits; and only where none does, no entry.
|
|
1362
|
+
if (fits && k === 0 && top > 0 && step.leaveAt?.floor && step.thin) {
|
|
1363
|
+
const { level } = step.leaveAt;
|
|
1364
|
+
const inner = levelOf(step.thin[level]).capped;
|
|
1365
|
+
let got;
|
|
1366
|
+
if (inner !== undefined) {
|
|
1367
|
+
const ls = (c) => at(step.thin?.[level], c, level, undefined, true);
|
|
1368
|
+
if (size(ls(1), 1, base) <= STRUCTURED_BUDGET) {
|
|
1369
|
+
let c = 1;
|
|
1370
|
+
let hi = Math.max(1, ...step.lists.map((p) => longestInside(listOf(data, p.split(".")), inner))) - 1;
|
|
1371
|
+
while (c < hi) {
|
|
1372
|
+
const mid = Math.ceil((c + hi) / 2);
|
|
1373
|
+
if (size(ls(mid), 1, base) <= STRUCTURED_BUDGET)
|
|
1374
|
+
c = mid;
|
|
1375
|
+
else
|
|
1376
|
+
hi = mid - 1;
|
|
1377
|
+
}
|
|
1378
|
+
got = ls(c);
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
for (let j = level + 1; !got && j < step.thin.length; j++) {
|
|
1382
|
+
const ls = at(step.thin[j], undefined, j);
|
|
1383
|
+
if (size(ls, 1, base) <= STRUCTURED_BUDGET)
|
|
1384
|
+
got = ls;
|
|
1385
|
+
}
|
|
1386
|
+
if (got) {
|
|
1387
|
+
lists = got;
|
|
1388
|
+
k = 1;
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
524
1391
|
}
|
|
525
1392
|
for (const list of lists)
|
|
526
|
-
kept.set(list.path, { list, k: Math.min(k, list.n), step: s });
|
|
1393
|
+
kept.set(list.path, { list, k: Math.min(k, list.n), step: s, ...(gone ? { gone: true } : {}) });
|
|
527
1394
|
if (fits)
|
|
528
1395
|
break;
|
|
529
1396
|
}
|
|
1397
|
+
const exact = () => {
|
|
1398
|
+
const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, false, fit);
|
|
1399
|
+
return JSON.stringify(structured(apply(), truncated, said)).length;
|
|
1400
|
+
};
|
|
1401
|
+
// #3139: every step cut and the result still past the budget: a keepAll step's entries leave now,
|
|
1402
|
+
// the step's first, as many as must, the rest of data as the steps after it left it. The steps
|
|
1403
|
+
// after it then take back what room is left (below), as though it were the step that fits.
|
|
1404
|
+
// #3139's review: with more than one keepAll step, the last one's entries leave first
|
|
1405
|
+
// (cve_remediation's fixed_branches before any vendor advisory).
|
|
1406
|
+
if (!fits) {
|
|
1407
|
+
for (const [s, step] of [...steps.entries()].reverse()) {
|
|
1408
|
+
if (!step.keepAll || fits)
|
|
1409
|
+
continue;
|
|
1410
|
+
const entries = [...kept.values()].filter((e) => e.step === s);
|
|
1411
|
+
if (entries.length === 0)
|
|
1412
|
+
continue;
|
|
1413
|
+
const set = (k) => {
|
|
1414
|
+
for (const e of entries)
|
|
1415
|
+
e.k = Math.min(k, e.list.n);
|
|
1416
|
+
};
|
|
1417
|
+
set(0);
|
|
1418
|
+
if (exact() > STRUCTURED_BUDGET)
|
|
1419
|
+
continue;
|
|
1420
|
+
let k = 0;
|
|
1421
|
+
let hi = Math.max(...entries.map((e) => e.list.n));
|
|
1422
|
+
while (k < hi) {
|
|
1423
|
+
const mid = Math.ceil((k + hi) / 2);
|
|
1424
|
+
set(mid);
|
|
1425
|
+
if (exact() <= STRUCTURED_BUDGET)
|
|
1426
|
+
k = mid;
|
|
1427
|
+
else
|
|
1428
|
+
hi = mid - 1;
|
|
1429
|
+
}
|
|
1430
|
+
set(k);
|
|
1431
|
+
for (const e of entries)
|
|
1432
|
+
e.step = steps.length;
|
|
1433
|
+
fits = true;
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
530
1436
|
// The room the step that fits leaves goes back to the steps emptied before it, the last first,
|
|
531
1437
|
// each step's lists keeping at most the same number of their entries, as many as fit, at the
|
|
532
1438
|
// level they were thinned to (scan_manifest's not_checked, emptied while its rows were whole,
|
|
533
1439
|
// keeps its entries once the thinned rows fit).
|
|
534
1440
|
if (fits) {
|
|
535
|
-
const exact = () => {
|
|
536
|
-
const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, false, wholeFit);
|
|
537
|
-
return JSON.stringify(structured(apply(), truncated, said)).length;
|
|
538
|
-
};
|
|
539
1441
|
const last = Math.max(...[...kept.values()].map((e) => e.step));
|
|
540
1442
|
for (let s = last - 1; s >= 0; s--) {
|
|
541
1443
|
const entries = [...kept.values()].filter((e) => e.step === s);
|
|
@@ -547,6 +1449,13 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
547
1449
|
};
|
|
548
1450
|
let k = Math.min(...entries.map((e) => e.k));
|
|
549
1451
|
let hi = Math.max(...entries.map((e) => e.list.n));
|
|
1452
|
+
if (steps[s].whole) {
|
|
1453
|
+
// Every entry or none (#3132's review).
|
|
1454
|
+
at(hi);
|
|
1455
|
+
if (exact() > STRUCTURED_BUDGET)
|
|
1456
|
+
at(k);
|
|
1457
|
+
continue;
|
|
1458
|
+
}
|
|
550
1459
|
while (k < hi) {
|
|
551
1460
|
const mid = Math.ceil((k + hi) / 2);
|
|
552
1461
|
at(mid);
|
|
@@ -558,7 +1467,102 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
558
1467
|
at(k);
|
|
559
1468
|
}
|
|
560
1469
|
}
|
|
561
|
-
|
|
1470
|
+
// #3132's review: then a giveBack step's entries, in their order in data, take back detail (see
|
|
1471
|
+
// DataCutStep): each the richest level above its own that fits. A RowKeep level is tried whole;
|
|
1472
|
+
// of the capped levels, only the leanest at or above the entry's own (check_sbom's compact one,
|
|
1473
|
+
// or the entry's own), uncapped and then at the most inner entries that fit, so a row takes back
|
|
1474
|
+
// more of its matches compact before fewer of them thinned. #3132's second review: an entry that
|
|
1475
|
+
// takes back nothing is passed, not an end (through round 2 it ended give-back, so a row at the
|
|
1476
|
+
// compact level holding every match, too large to move up, kept every later capped row at its
|
|
1477
|
+
// cap while room was left); the entries that took some are named by position (givenBackSaid).
|
|
1478
|
+
if (fits) {
|
|
1479
|
+
for (const [s, step] of steps.entries()) {
|
|
1480
|
+
if (!step.giveBack || !step.thin)
|
|
1481
|
+
continue;
|
|
1482
|
+
const thin = step.thin;
|
|
1483
|
+
for (const e of [...kept.values()].filter((x) => x.step === s)) {
|
|
1484
|
+
const { list } = e;
|
|
1485
|
+
const own = list.level;
|
|
1486
|
+
// #3269's second review: the one entry kept below its floor takes back nothing (more did not fit).
|
|
1487
|
+
if (own === undefined || list.thin === undefined || list.below)
|
|
1488
|
+
continue;
|
|
1489
|
+
const whole = listOf(data, list.at);
|
|
1490
|
+
if (!Array.isArray(whole))
|
|
1491
|
+
continue;
|
|
1492
|
+
let leanestCapped = -1;
|
|
1493
|
+
for (let j = 0; j <= own; j++)
|
|
1494
|
+
if (levelOf(thin[j]).capped !== undefined)
|
|
1495
|
+
leanestCapped = j;
|
|
1496
|
+
const tryAt = (i, j, c) => {
|
|
1497
|
+
const l = levelOf(thin[j]);
|
|
1498
|
+
list.over.set(i, { entry: thinned(whole[i], thin[j], c), level: j, cap: c, keep: l.keep, ...(l.said && l.capped && c < Number.POSITIVE_INFINITY ? { said: l.said(c) } : {}) });
|
|
1499
|
+
if (exact() <= STRUCTURED_BUDGET)
|
|
1500
|
+
return true;
|
|
1501
|
+
list.over.delete(i);
|
|
1502
|
+
return false;
|
|
1503
|
+
};
|
|
1504
|
+
for (const i of list.kept(e.k)) {
|
|
1505
|
+
// #3269: an entry outFirst cut apart takes back nothing (the others come first).
|
|
1506
|
+
if (list.outThin && list.isNamed(i))
|
|
1507
|
+
continue;
|
|
1508
|
+
// An entry its own level already keeps as the richest would is passed, not an end.
|
|
1509
|
+
if (JSON.stringify(thinned(whole[i], thin[0])) === JSON.stringify(thinned(whole[i], thin[own], list.cap)))
|
|
1510
|
+
continue;
|
|
1511
|
+
let took = false;
|
|
1512
|
+
for (let j = 0; j <= own && !took; j++) {
|
|
1513
|
+
const l = levelOf(thin[j]);
|
|
1514
|
+
const inner = innerOf(whole[i], l.capped);
|
|
1515
|
+
if (l.capped === undefined) {
|
|
1516
|
+
if (j < own)
|
|
1517
|
+
took = tryAt(i, j, Number.POSITIVE_INFINITY);
|
|
1518
|
+
continue;
|
|
1519
|
+
}
|
|
1520
|
+
if (j !== leanestCapped)
|
|
1521
|
+
continue;
|
|
1522
|
+
// #3269's second review: the cap the entry is cut to, its floor where that is more.
|
|
1523
|
+
const now = capFor(l, whole[i], list.cap);
|
|
1524
|
+
if (j < own || now < inner)
|
|
1525
|
+
took = tryAt(i, j, Number.POSITIVE_INFINITY);
|
|
1526
|
+
if (took)
|
|
1527
|
+
break;
|
|
1528
|
+
let c = j === own ? now + 1 : capFor(l, whole[i], l.least ?? 0);
|
|
1529
|
+
let hi = inner - 1;
|
|
1530
|
+
if (c > hi || !tryAt(i, j, c))
|
|
1531
|
+
continue;
|
|
1532
|
+
while (c < hi) {
|
|
1533
|
+
const mid = Math.ceil((c + hi) / 2);
|
|
1534
|
+
if (tryAt(i, j, mid))
|
|
1535
|
+
c = mid;
|
|
1536
|
+
else
|
|
1537
|
+
hi = mid - 1;
|
|
1538
|
+
}
|
|
1539
|
+
took = tryAt(i, j, c);
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
}
|
|
1545
|
+
// #3269: last, the entries outFirst sent out come back with the room left, cut to its level, the
|
|
1546
|
+
// last to leave first (they leave first of all, so they come back only where every other entry
|
|
1547
|
+
// stayed), as many as fit.
|
|
1548
|
+
if (fits) {
|
|
1549
|
+
for (const e of kept.values()) {
|
|
1550
|
+
if (!e.gone || e.k < e.list.n - e.list.named)
|
|
1551
|
+
continue;
|
|
1552
|
+
let hi = e.list.n;
|
|
1553
|
+
let k = e.k;
|
|
1554
|
+
while (k < hi) {
|
|
1555
|
+
const mid = Math.ceil((k + hi) / 2);
|
|
1556
|
+
e.k = mid;
|
|
1557
|
+
if (exact() <= STRUCTURED_BUDGET)
|
|
1558
|
+
k = mid;
|
|
1559
|
+
else
|
|
1560
|
+
hi = mid - 1;
|
|
1561
|
+
}
|
|
1562
|
+
e.k = k;
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, !fits, fit);
|
|
562
1566
|
if (truncated === undefined)
|
|
563
1567
|
return undefined;
|
|
564
1568
|
const d = apply();
|