echelongraph-mcp 3.0.0 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +23 -7
- package/dist/fixStatus.js +4 -2
- package/dist/httpPolicy.js +16 -1
- package/dist/index.js +66 -9
- package/dist/prompts.js +8 -2
- package/dist/requiredText.js +15 -0
- package/dist/textBudget.js +281 -27
- package/dist/tools/check_affected.js +4 -1
- package/dist/tools/check_sbom.js +114 -4
- package/dist/tools/cve_intel.js +49 -4
- package/dist/tools/cve_remediation.js +82 -8
- package/dist/tools/epss_history.js +4 -1
- package/dist/tools/kev_recent.js +4 -1
- package/dist/tools/match_batch.js +22 -6
- package/dist/tools/scan_manifest.js +15 -3
- package/dist/tools/vendor_advisories.js +13 -4
- package/docs/tools.md +10 -6
- package/package.json +1 -1
- package/server.json +2 -2
package/dist/textBudget.js
CHANGED
|
@@ -62,7 +62,11 @@ import * as z from "zod";
|
|
|
62
62
|
// the tool names leave first (the clean rows, counted in data.summary), then the last of the
|
|
63
63
|
// rest, or for a step of series named keepNewest (exposure_radar's daily and weekly counts,
|
|
64
64
|
// which the API sends oldest first) its last ones, the newest, the oldest leaving first; a
|
|
65
|
-
// step that cannot fit even empty is emptied, and the next one is cut
|
|
65
|
+
// step that cannot fit even empty is emptied, and the next one is cut. A level can cut inside
|
|
66
|
+
// each entry on its own terms (an EntryLevel, #3132: check_sbom's rows keep at most the same
|
|
67
|
+
// number of matches, the most that fit, so only the rows with more lose any), and a step named
|
|
68
|
+
// keepAll (#3139: cve_remediation's vendor_remediations) keeps every entry at its leanest level
|
|
69
|
+
// while the later steps are cut, its entries leaving only once they all are;
|
|
66
70
|
// 2. then every other list at the top level of data, the same way. A list inside an object of
|
|
67
71
|
// data (get_cve's remediation summary, with fix_status, #2841) is cut only where a DataCut
|
|
68
72
|
// names it, and none names get_cve's. Once a step fits, the room it leaves goes back to the
|
|
@@ -70,7 +74,9 @@ import * as z from "zod";
|
|
|
70
74
|
// emptied while its rows were whole, keeps its entries once the thinned rows fit);
|
|
71
75
|
// 3. no string is shortened and no value changes: a thinned entry keeps some of its fields as
|
|
72
76
|
// they are (every field a thinning level drops is optional in the outputSchema, so data still
|
|
73
|
-
// validates against it), and no field outside the lists cut changes
|
|
77
|
+
// validates against it), and no field outside the lists cut changes (the one field a cut
|
|
78
|
+
// adds is an EntryLevel's mark, check_sbom's matches_truncated, which says what the row's own
|
|
79
|
+
// cut left out, and which its outputSchema names); the envelope says
|
|
74
80
|
// data_truncated true, truncated gives each list cut by its path in data with its total, the
|
|
75
81
|
// number returned and, where its entries are thinned, thinned true, and the note ends with
|
|
76
82
|
// sentences starting DATA CUT that say what was cut and to which fields, how to read the rest
|
|
@@ -118,11 +124,12 @@ function cutValue(v, clip, cap, s) {
|
|
|
118
124
|
if (typeof v === "string") {
|
|
119
125
|
if (v.length <= clip)
|
|
120
126
|
return v;
|
|
121
|
-
s
|
|
127
|
+
if (s)
|
|
128
|
+
s.clipped++;
|
|
122
129
|
return clipString(v, clip);
|
|
123
130
|
}
|
|
124
131
|
if (Array.isArray(v)) {
|
|
125
|
-
if (v.length > cap)
|
|
132
|
+
if (s && v.length > cap)
|
|
126
133
|
s.capped++;
|
|
127
134
|
return v.slice(0, cap).map((x) => cutValue(x, clip, cap, s));
|
|
128
135
|
}
|
|
@@ -130,18 +137,31 @@ function cutValue(v, clip, cap, s) {
|
|
|
130
137
|
return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, cutValue(x, clip, cap, s)]));
|
|
131
138
|
return v;
|
|
132
139
|
}
|
|
140
|
+
// A RowKeep as the fields it names, each with its subfields (null: kept whole), built once per
|
|
141
|
+
// RowKeep (3.0.2: through 3.0.1, once per row cut; a 2,000-row answer's structured cut built it
|
|
142
|
+
// some 12,000 times). A RowKeep is never changed once built.
|
|
143
|
+
const wantedOf = (() => {
|
|
144
|
+
const built = new WeakMap();
|
|
145
|
+
return (keep) => {
|
|
146
|
+
let wanted = built.get(keep);
|
|
147
|
+
if (!wanted)
|
|
148
|
+
built.set(keep, (wanted = new Map(keep.map((k) => (typeof k === "string" ? [k, null] : [k[0], k[1]])))));
|
|
149
|
+
return wanted;
|
|
150
|
+
};
|
|
151
|
+
})();
|
|
133
152
|
function cutRow(row, keep, level, s, prefix = "") {
|
|
134
|
-
const wanted =
|
|
153
|
+
const wanted = wantedOf(keep);
|
|
135
154
|
const cap = level.cap ?? Number.POSITIVE_INFINITY;
|
|
136
155
|
const out = {};
|
|
137
|
-
for (const
|
|
156
|
+
for (const k of Object.keys(row)) {
|
|
157
|
+
const v = row[k];
|
|
138
158
|
if (!wanted.has(k)) {
|
|
139
|
-
s
|
|
159
|
+
s?.left.add(`${prefix}${k}`);
|
|
140
160
|
continue;
|
|
141
161
|
}
|
|
142
162
|
const sub = wanted.get(k);
|
|
143
163
|
if (sub && Array.isArray(v)) {
|
|
144
|
-
if (v.length > cap)
|
|
164
|
+
if (s && v.length > cap)
|
|
145
165
|
s.capped++;
|
|
146
166
|
out[k] = v.slice(0, cap).map((e) => (isPlainObject(e) ? cutRow(e, sub, level, s, `${prefix}${k}[].`) : cutValue(e, level.clip, cap, s)));
|
|
147
167
|
}
|
|
@@ -336,18 +356,51 @@ export const answeringNow = () => calls.getStore();
|
|
|
336
356
|
// tool and directs no model (test/description-policy.test.mjs reads inputSchema descriptions too).
|
|
337
357
|
export const FULL_ARG_DESCRIPTION = "true: data in the structured result is the answer whole, however long. Without it, a structured result past 30,000 characters of JSON has its data cut (DATA CUT): in an order the note states, the entries of its longest lists keep only some of their fields, and then the lists only some of their entries; data_truncated is true, and truncated gives each list cut with its total, the number returned and whether its entries are thinned. Whole data is for a program; a client that sizes structured results, as Claude Code does against MAX_MCP_OUTPUT_TOKENS, can refuse an answer past its cap.";
|
|
338
358
|
export const FULL_ARG = z.boolean().optional().describe(FULL_ARG_DESCRIPTION);
|
|
339
|
-
|
|
359
|
+
// A level as its fields, and what it does inside each entry (#3132).
|
|
360
|
+
const levelOf = (l) => (Array.isArray(l) ? { keep: l } : l);
|
|
361
|
+
const thinned = (e, level, cap = Number.POSITIVE_INFINITY) => {
|
|
362
|
+
if (!isPlainObject(e))
|
|
363
|
+
return e;
|
|
364
|
+
const { keep, capped, mark } = levelOf(level);
|
|
365
|
+
const t = cutRow(e, keep, { keep, clip: Number.POSITIVE_INFINITY });
|
|
366
|
+
const inner = capped ? t[capped] : undefined;
|
|
367
|
+
if (capped && Array.isArray(inner) && inner.length > cap)
|
|
368
|
+
t[capped] = inner.slice(0, cap);
|
|
369
|
+
return mark ? { ...t, ...mark(e, cap) } : t;
|
|
370
|
+
};
|
|
371
|
+
// The longest list named `capped` inside any entry of a list (#3132): the cap a level starts below.
|
|
372
|
+
const longestInside = (v, capped) => Array.isArray(v) ? v.reduce((m, e) => Math.max(m, Array.isArray(field(e, capped)) ? field(e, capped).length : 0), 0) : 0;
|
|
340
373
|
// An entry JSON cannot write (undefined) is written null in a list.
|
|
341
374
|
const entryChars = (e) => JSON.stringify(e)?.length ?? 4;
|
|
342
375
|
const listOf = (data, at) => at.reduce((o, k) => field(o, k), data);
|
|
343
|
-
|
|
376
|
+
// 3.0.2: a list's entries are measured, thinned, only when the cut first asks how many characters
|
|
377
|
+
// some of them take (chars), each thinned, measured and let go; and thinned to be kept only when
|
|
378
|
+
// the cut asks for them (keep), only those it keeps, each once. Through 3.0.1 every list the cut
|
|
379
|
+
// tried held every entry thinned for as long as the cut ran, and a level that caps a list inside
|
|
380
|
+
// each entry thinned every entry at no cap before it tried any: on a 2,000-row check_sbom answer
|
|
381
|
+
// (test/http-memory.test.mjs's) the structured cut allocated 108 MB, against 18 MB on 3.0.0 and 24
|
|
382
|
+
// MB now (V8's sampling heap profiler, collected objects included). What is measured and kept is
|
|
383
|
+
// the same: each entry thinned as before, its characters summed as before.
|
|
384
|
+
function listAt(data, path, firstOut, level, newest, cap = Number.POSITIVE_INFINITY, levelAt) {
|
|
344
385
|
const at = path.split(".");
|
|
345
386
|
const v = listOf(data, at);
|
|
346
387
|
if (!Array.isArray(v) || v.length === 0)
|
|
347
388
|
return undefined;
|
|
348
389
|
const n = v.length;
|
|
349
|
-
const
|
|
350
|
-
const
|
|
390
|
+
const made = [];
|
|
391
|
+
const entry = (i) => {
|
|
392
|
+
if (!level)
|
|
393
|
+
return v[i];
|
|
394
|
+
if (!(i in made))
|
|
395
|
+
made[i] = thinned(v[i], level, cap);
|
|
396
|
+
return made[i];
|
|
397
|
+
};
|
|
398
|
+
const thin = level ? levelOf(level).keep : undefined;
|
|
399
|
+
// A capped level's clause only where its cap cuts an entry's list (#3139's review: a package
|
|
400
|
+
// with one range is not "at most its first 1").
|
|
401
|
+
const lv = level ? levelOf(level) : undefined;
|
|
402
|
+
const bites = lv?.capped === undefined || v.some((e) => (Array.isArray(field(e, lv.capped)) ? field(e, lv.capped).length : 0) > cap);
|
|
403
|
+
const thinSaid = lv && bites ? lv.said?.(cap) : undefined;
|
|
351
404
|
// The indices in the order they leave: from the end, or, for a series kept newest, from the start.
|
|
352
405
|
const leavingOrder = (i) => (newest ? i : n - 1 - i);
|
|
353
406
|
const order = [];
|
|
@@ -360,24 +413,43 @@ function listAt(data, path, firstOut, thin, newest) {
|
|
|
360
413
|
for (let j = 0; j < n; j++)
|
|
361
414
|
if (!queued.has(leavingOrder(j)))
|
|
362
415
|
order.push(leavingOrder(j));
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
416
|
+
// The characters of the entries in the order they leave, summed: leaving[j] is the first j's.
|
|
417
|
+
let leaving;
|
|
418
|
+
const sums = () => {
|
|
419
|
+
if (leaving)
|
|
420
|
+
return leaving;
|
|
421
|
+
const lens = [];
|
|
422
|
+
for (let i = 0; i < n; i++)
|
|
423
|
+
lens.push(entryChars(!level || i in made ? entry(i) : thinned(v[i], level, cap)));
|
|
424
|
+
leaving = [0];
|
|
425
|
+
for (const i of order)
|
|
426
|
+
leaving.push(leaving[leaving.length - 1] + lens[i]);
|
|
427
|
+
return leaving;
|
|
428
|
+
};
|
|
367
429
|
const kept = (k) => Math.max(0, Math.min(n, k));
|
|
430
|
+
const over = new Map();
|
|
431
|
+
const keptAt = (k) => {
|
|
432
|
+
const out = new Set(order.slice(0, n - kept(k)));
|
|
433
|
+
return Array.from({ length: n }, (_, i) => i).filter((i) => !out.has(i));
|
|
434
|
+
};
|
|
368
435
|
return {
|
|
369
436
|
path,
|
|
370
437
|
at,
|
|
371
438
|
n,
|
|
372
439
|
firstOut,
|
|
373
440
|
thin,
|
|
441
|
+
...(thinSaid ? { thinSaid } : {}),
|
|
374
442
|
...(newest ? { newest } : {}),
|
|
375
443
|
named,
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
444
|
+
...(levelAt !== undefined ? { level: levelAt } : {}),
|
|
445
|
+
cap,
|
|
446
|
+
over,
|
|
447
|
+
chars: (k) => {
|
|
448
|
+
const l = sums();
|
|
449
|
+
return l[n] - l[n - kept(k)] + Math.max(0, kept(k) - 1);
|
|
380
450
|
},
|
|
451
|
+
kept: keptAt,
|
|
452
|
+
keep: (k) => keptAt(k).map((i) => (over.has(i) ? over.get(i)?.entry : entry(i))),
|
|
381
453
|
};
|
|
382
454
|
}
|
|
383
455
|
// `o` with the list at `at` replaced by `v`, each object on the way copied, its keys in their order.
|
|
@@ -388,6 +460,30 @@ function withList(o, at, v) {
|
|
|
388
460
|
const orListed = (xs) => (xs.length < 2 ? xs.join("") : `${xs.slice(0, -1).join(", ")} or ${xs[xs.length - 1]}`);
|
|
389
461
|
// The fields a thinning level keeps, as a sentence says them.
|
|
390
462
|
const keepSaid = (keep) => listed(keep.map((x) => (typeof x === "string" ? x : `${x[0]} (each with ${listed([...x[1]])})`)));
|
|
463
|
+
// The clause on the entries that took back detail (#3132's review): their positions among those
|
|
464
|
+
// kept, in their order in data, grouped where one level and cap holds a run of them (#3132's second
|
|
465
|
+
// review: not always the first kept, since give-back passes an entry that cannot take any back).
|
|
466
|
+
function givenBackSaid(list, k) {
|
|
467
|
+
if (list.over.size === 0)
|
|
468
|
+
return "";
|
|
469
|
+
const kept = list.kept(k);
|
|
470
|
+
const runs = [];
|
|
471
|
+
kept.forEach((i, pos) => {
|
|
472
|
+
const g = list.over.get(i);
|
|
473
|
+
if (!g)
|
|
474
|
+
return;
|
|
475
|
+
const last = runs[runs.length - 1];
|
|
476
|
+
if (last && last.to === pos && last.g.level === g.level && last.g.cap === g.cap)
|
|
477
|
+
last.to = pos + 1;
|
|
478
|
+
else
|
|
479
|
+
runs.push({ from: pos + 1, to: pos + 1, g });
|
|
480
|
+
});
|
|
481
|
+
if (runs.length === 0)
|
|
482
|
+
return "";
|
|
483
|
+
const n = runs.reduce((t, r) => t + r.to - r.from + 1, 0);
|
|
484
|
+
const one = (r) => `${r.from === r.to ? `entry ${r.from}` : `entries ${r.from} to ${r.to}`} cut to ${keepSaid(r.g.keep)}${r.g.said ? `, ${r.g.said}` : ""}`;
|
|
485
|
+
return `; with the room left, ${n} of the entries kept take back detail (by position among those kept, in their order in data): ${runs.map(one).join("; ")}`;
|
|
486
|
+
}
|
|
391
487
|
// What a cut keeps of each list it has cut so far, as truncated and as the DATA CUT sentences.
|
|
392
488
|
function cutSaid(cut, kept, wholeChars, data, over, wholeFit) {
|
|
393
489
|
const isThinned = ({ list, k }) => list.thin !== undefined && k > 0;
|
|
@@ -405,7 +501,7 @@ function cutSaid(cut, kept, wholeChars, data, over, wholeFit) {
|
|
|
405
501
|
const which = list.firstOut ? `whose ${list.firstOut.field} is ${orListed(list.firstOut.values)} (${list.firstOut.why})` : "";
|
|
406
502
|
const leftOut = out === 0 ? "" : first === 0 ? `, the ${end} ${out} left out` : first === out ? `, the ${out} left out being entries ${which}` : `, the ${out} left out being the ${first} ${which} and the ${end} ${out - first} of the rest`;
|
|
407
503
|
if (isThinned(x)) {
|
|
408
|
-
parts.push(`${list.path} keeps ${k === list.n ? `all ${k}` : `${k} of its ${list.n}`} entries, each cut to ${keepSaid(list.thin)}${leftOut}`);
|
|
504
|
+
parts.push(`${list.path} keeps ${k === list.n ? `all ${k}` : `${k} of its ${list.n}`} entries, each cut to ${keepSaid(list.thin)}${list.thinSaid ? `, ${list.thinSaid}` : ""}${leftOut}${givenBackSaid(list, k)}`);
|
|
409
505
|
continue;
|
|
410
506
|
}
|
|
411
507
|
if (!list.firstOut || first === 0 || k === 0) {
|
|
@@ -491,16 +587,44 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
491
587
|
let fits = false;
|
|
492
588
|
for (const [s, step] of steps.entries()) {
|
|
493
589
|
// Every entry whole, then each thinning level in turn, every entry kept; at the leanest level
|
|
494
|
-
// the entries leave until the result fits.
|
|
590
|
+
// the entries leave until the result fits (with keepAll, #3139, not in this step: see below).
|
|
495
591
|
const levels = [undefined, ...(step.thin ?? [])];
|
|
496
592
|
let lists = [];
|
|
497
593
|
let k = 0;
|
|
594
|
+
const at = (level, cap, levelAt) => step.lists.map((p) => listAt(data, p, step.leaveOutFirst, level, step.keepNewest, cap, levelAt)).filter((l) => l !== undefined && !kept.has(l.path));
|
|
498
595
|
for (const [i, thin] of levels.entries()) {
|
|
499
|
-
lists =
|
|
596
|
+
lists = at(thin, undefined, i - 1);
|
|
500
597
|
if (lists.length === 0)
|
|
501
598
|
break;
|
|
502
599
|
const base = JSON.stringify(apply(lists.map((list) => ({ list, k: 0 })))).length;
|
|
503
600
|
const top = Math.max(...lists.map((l) => l.n));
|
|
601
|
+
// #3132: a level that caps a list inside each entry, every entry kept: the most entries of
|
|
602
|
+
// that inner list, the same number in each, that fit; a level where even none fits is passed.
|
|
603
|
+
// #3132's review: a level with `least` is taken at that cap or more, or passed.
|
|
604
|
+
const capped = thin === undefined ? undefined : levelOf(thin).capped;
|
|
605
|
+
if (capped !== undefined) {
|
|
606
|
+
const longest = Math.max(0, ...step.lists.map((p) => longestInside(listOf(data, p.split(".")), capped)));
|
|
607
|
+
const least = (thin === undefined ? undefined : levelOf(thin).least) ?? 0;
|
|
608
|
+
if (longest > least && size(at(thin, least, i - 1), top, base) <= STRUCTURED_BUDGET) {
|
|
609
|
+
let m = least;
|
|
610
|
+
let hi = longest - 1;
|
|
611
|
+
while (m < hi) {
|
|
612
|
+
const mid = Math.ceil((m + hi) / 2);
|
|
613
|
+
if (size(at(thin, mid, i - 1), top, base) <= STRUCTURED_BUDGET)
|
|
614
|
+
m = mid;
|
|
615
|
+
else
|
|
616
|
+
hi = mid - 1;
|
|
617
|
+
}
|
|
618
|
+
lists = at(thin, m, i - 1);
|
|
619
|
+
k = top;
|
|
620
|
+
fits = true;
|
|
621
|
+
break;
|
|
622
|
+
}
|
|
623
|
+
if (i < levels.length - 1)
|
|
624
|
+
continue;
|
|
625
|
+
// The last level, and even its least cap does not fit: the entries go on at that cap.
|
|
626
|
+
lists = at(thin, least, i - 1);
|
|
627
|
+
}
|
|
504
628
|
if (i < levels.length - 1) {
|
|
505
629
|
if (size(lists, top, base) <= STRUCTURED_BUDGET) {
|
|
506
630
|
k = top;
|
|
@@ -509,6 +633,25 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
509
633
|
}
|
|
510
634
|
continue;
|
|
511
635
|
}
|
|
636
|
+
if (step.keepAll) {
|
|
637
|
+
// #3139: every entry stays, at this level; the later steps are cut first.
|
|
638
|
+
k = top;
|
|
639
|
+
fits = size(lists, top, base) <= STRUCTURED_BUDGET;
|
|
640
|
+
break;
|
|
641
|
+
}
|
|
642
|
+
// #3132's review: entries leave at the step's leaveAt level, where it names one.
|
|
643
|
+
if (step.leaveAt && step.thin) {
|
|
644
|
+
const { level, cap } = step.leaveAt;
|
|
645
|
+
lists = at(step.thin[level], cap, level);
|
|
646
|
+
}
|
|
647
|
+
if (step.whole) {
|
|
648
|
+
// Every entry or none (#3132's review: data.not_sent_purls).
|
|
649
|
+
if (size(lists, 0, base) <= STRUCTURED_BUDGET) {
|
|
650
|
+
k = size(lists, top, base) <= STRUCTURED_BUDGET ? top : 0;
|
|
651
|
+
fits = true;
|
|
652
|
+
}
|
|
653
|
+
break;
|
|
654
|
+
}
|
|
512
655
|
if (size(lists, 0, base) <= STRUCTURED_BUDGET) {
|
|
513
656
|
// The most entries each list of the step keeps with the result inside the budget.
|
|
514
657
|
let hi = top;
|
|
@@ -527,15 +670,50 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
527
670
|
if (fits)
|
|
528
671
|
break;
|
|
529
672
|
}
|
|
673
|
+
const exact = () => {
|
|
674
|
+
const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, false, wholeFit);
|
|
675
|
+
return JSON.stringify(structured(apply(), truncated, said)).length;
|
|
676
|
+
};
|
|
677
|
+
// #3139: every step cut and the result still past the budget: a keepAll step's entries leave now,
|
|
678
|
+
// the step's first, as many as must, the rest of data as the steps after it left it. The steps
|
|
679
|
+
// after it then take back what room is left (below), as though it were the step that fits.
|
|
680
|
+
// #3139's review: with more than one keepAll step, the last one's entries leave first
|
|
681
|
+
// (cve_remediation's fixed_branches before any vendor advisory).
|
|
682
|
+
if (!fits) {
|
|
683
|
+
for (const [s, step] of [...steps.entries()].reverse()) {
|
|
684
|
+
if (!step.keepAll || fits)
|
|
685
|
+
continue;
|
|
686
|
+
const entries = [...kept.values()].filter((e) => e.step === s);
|
|
687
|
+
if (entries.length === 0)
|
|
688
|
+
continue;
|
|
689
|
+
const set = (k) => {
|
|
690
|
+
for (const e of entries)
|
|
691
|
+
e.k = Math.min(k, e.list.n);
|
|
692
|
+
};
|
|
693
|
+
set(0);
|
|
694
|
+
if (exact() > STRUCTURED_BUDGET)
|
|
695
|
+
continue;
|
|
696
|
+
let k = 0;
|
|
697
|
+
let hi = Math.max(...entries.map((e) => e.list.n));
|
|
698
|
+
while (k < hi) {
|
|
699
|
+
const mid = Math.ceil((k + hi) / 2);
|
|
700
|
+
set(mid);
|
|
701
|
+
if (exact() <= STRUCTURED_BUDGET)
|
|
702
|
+
k = mid;
|
|
703
|
+
else
|
|
704
|
+
hi = mid - 1;
|
|
705
|
+
}
|
|
706
|
+
set(k);
|
|
707
|
+
for (const e of entries)
|
|
708
|
+
e.step = steps.length;
|
|
709
|
+
fits = true;
|
|
710
|
+
}
|
|
711
|
+
}
|
|
530
712
|
// The room the step that fits leaves goes back to the steps emptied before it, the last first,
|
|
531
713
|
// each step's lists keeping at most the same number of their entries, as many as fit, at the
|
|
532
714
|
// level they were thinned to (scan_manifest's not_checked, emptied while its rows were whole,
|
|
533
715
|
// keeps its entries once the thinned rows fit).
|
|
534
716
|
if (fits) {
|
|
535
|
-
const exact = () => {
|
|
536
|
-
const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, false, wholeFit);
|
|
537
|
-
return JSON.stringify(structured(apply(), truncated, said)).length;
|
|
538
|
-
};
|
|
539
717
|
const last = Math.max(...[...kept.values()].map((e) => e.step));
|
|
540
718
|
for (let s = last - 1; s >= 0; s--) {
|
|
541
719
|
const entries = [...kept.values()].filter((e) => e.step === s);
|
|
@@ -547,6 +725,13 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
547
725
|
};
|
|
548
726
|
let k = Math.min(...entries.map((e) => e.k));
|
|
549
727
|
let hi = Math.max(...entries.map((e) => e.list.n));
|
|
728
|
+
if (steps[s].whole) {
|
|
729
|
+
// Every entry or none (#3132's review).
|
|
730
|
+
at(hi);
|
|
731
|
+
if (exact() > STRUCTURED_BUDGET)
|
|
732
|
+
at(k);
|
|
733
|
+
continue;
|
|
734
|
+
}
|
|
550
735
|
while (k < hi) {
|
|
551
736
|
const mid = Math.ceil((k + hi) / 2);
|
|
552
737
|
at(mid);
|
|
@@ -558,6 +743,75 @@ export function cutData(data, cut, wholeChars, structured) {
|
|
|
558
743
|
at(k);
|
|
559
744
|
}
|
|
560
745
|
}
|
|
746
|
+
// #3132's review: then a giveBack step's entries, in their order in data, take back detail (see
|
|
747
|
+
// DataCutStep): each the richest level above its own that fits. A RowKeep level is tried whole;
|
|
748
|
+
// of the capped levels, only the leanest at or above the entry's own (check_sbom's compact one,
|
|
749
|
+
// or the entry's own), uncapped and then at the most inner entries that fit, so a row takes back
|
|
750
|
+
// more of its matches compact before fewer of them thinned. #3132's second review: an entry that
|
|
751
|
+
// takes back nothing is passed, not an end (through round 2 it ended give-back, so a row at the
|
|
752
|
+
// compact level holding every match, too large to move up, kept every later capped row at its
|
|
753
|
+
// cap while room was left); the entries that took some are named by position (givenBackSaid).
|
|
754
|
+
if (fits) {
|
|
755
|
+
for (const [s, step] of steps.entries()) {
|
|
756
|
+
if (!step.giveBack || !step.thin)
|
|
757
|
+
continue;
|
|
758
|
+
const thin = step.thin;
|
|
759
|
+
for (const e of [...kept.values()].filter((x) => x.step === s)) {
|
|
760
|
+
const { list } = e;
|
|
761
|
+
const own = list.level;
|
|
762
|
+
if (own === undefined || list.thin === undefined)
|
|
763
|
+
continue;
|
|
764
|
+
const whole = listOf(data, list.at);
|
|
765
|
+
if (!Array.isArray(whole))
|
|
766
|
+
continue;
|
|
767
|
+
let leanestCapped = -1;
|
|
768
|
+
for (let j = 0; j <= own; j++)
|
|
769
|
+
if (levelOf(thin[j]).capped !== undefined)
|
|
770
|
+
leanestCapped = j;
|
|
771
|
+
const tryAt = (i, j, c) => {
|
|
772
|
+
const l = levelOf(thin[j]);
|
|
773
|
+
list.over.set(i, { entry: thinned(whole[i], thin[j], c), level: j, cap: c, keep: l.keep, ...(l.said && l.capped && c < Number.POSITIVE_INFINITY ? { said: l.said(c) } : {}) });
|
|
774
|
+
if (exact() <= STRUCTURED_BUDGET)
|
|
775
|
+
return true;
|
|
776
|
+
list.over.delete(i);
|
|
777
|
+
return false;
|
|
778
|
+
};
|
|
779
|
+
for (const i of list.kept(e.k)) {
|
|
780
|
+
// An entry its own level already keeps as the richest would is passed, not an end.
|
|
781
|
+
if (JSON.stringify(thinned(whole[i], thin[0])) === JSON.stringify(thinned(whole[i], thin[own], list.cap)))
|
|
782
|
+
continue;
|
|
783
|
+
let took = false;
|
|
784
|
+
for (let j = 0; j <= own && !took; j++) {
|
|
785
|
+
const l = levelOf(thin[j]);
|
|
786
|
+
const inner = l.capped !== undefined && Array.isArray(field(whole[i], l.capped)) ? field(whole[i], l.capped).length : 0;
|
|
787
|
+
if (l.capped === undefined) {
|
|
788
|
+
if (j < own)
|
|
789
|
+
took = tryAt(i, j, Number.POSITIVE_INFINITY);
|
|
790
|
+
continue;
|
|
791
|
+
}
|
|
792
|
+
if (j !== leanestCapped)
|
|
793
|
+
continue;
|
|
794
|
+
if (j < own || list.cap < inner)
|
|
795
|
+
took = tryAt(i, j, Number.POSITIVE_INFINITY);
|
|
796
|
+
if (took)
|
|
797
|
+
break;
|
|
798
|
+
let c = j === own ? list.cap + 1 : (l.least ?? 0);
|
|
799
|
+
let hi = inner - 1;
|
|
800
|
+
if (c > hi || !tryAt(i, j, c))
|
|
801
|
+
continue;
|
|
802
|
+
while (c < hi) {
|
|
803
|
+
const mid = Math.ceil((c + hi) / 2);
|
|
804
|
+
if (tryAt(i, j, mid))
|
|
805
|
+
c = mid;
|
|
806
|
+
else
|
|
807
|
+
hi = mid - 1;
|
|
808
|
+
}
|
|
809
|
+
took = tryAt(i, j, c);
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
}
|
|
561
815
|
const { truncated, said } = cutSaid(cut, [...kept.values()], wholeChars, data, !fits, wholeFit);
|
|
562
816
|
if (truncated === undefined)
|
|
563
817
|
return undefined;
|
|
@@ -373,8 +373,11 @@ async function checkAffected(kit, a) {
|
|
|
373
373
|
return kit.crashed(TOOL, e);
|
|
374
374
|
}
|
|
375
375
|
}
|
|
376
|
+
// One output schema per process, not per request (3.0.2; index.ts envelopeSchema says why).
|
|
377
|
+
const OUTPUTS = new WeakMap();
|
|
376
378
|
export function registerCheckAffected(server, kit) {
|
|
377
|
-
const output = kit.envelopeSchema({ data: DATA, coverage: COVERAGE, freshness: null });
|
|
379
|
+
const output = OUTPUTS.get(kit.envelopeSchema) ?? kit.envelopeSchema({ data: DATA, coverage: COVERAGE, freshness: null });
|
|
380
|
+
OUTPUTS.set(kit.envelopeSchema, output);
|
|
378
381
|
server.registerTool(TOOL, {
|
|
379
382
|
title: CHECK_AFFECTED_TITLE,
|
|
380
383
|
description: CHECK_AFFECTED_DESCRIPTION,
|
package/dist/tools/check_sbom.js
CHANGED
|
@@ -100,15 +100,121 @@ export const SBOM_ROW_THIN = [
|
|
|
100
100
|
"cve_ids",
|
|
101
101
|
"matched_package",
|
|
102
102
|
"matched_via",
|
|
103
|
+
"matched_version_from",
|
|
104
|
+
"purl_name",
|
|
103
105
|
["matches", ["cve_id", "kev_listed", "ransomware", "epss_score", "effective_score", "score_assessed", "fixed_in", "fixed_in_reason"]],
|
|
104
106
|
];
|
|
107
|
+
// #3132: past SBOM_ROW_THIN, each row is cut on its own terms before any row leaves. Through 3.0.0
|
|
108
|
+
// SBOM_ROW_THIN was the only level, and it keeps every match object, so one row still large once
|
|
109
|
+
// thinned left data, and every row after it with it (rows leave from the end): measured on hosted
|
|
110
|
+
// 3.0.0 (2026-10-11), five Debian 12 purls, all affected, kept 1 row (libssl3; linux-libc-dev
|
|
111
|
+
// 6.1.55-1's 200 matches are 42,417 characters thinned); Juice Shop 11.1.2 with
|
|
112
|
+
// magento/community-edition 2.3.0 inserted at index 2 kept 2 of 51 rows (19 affected); that purl
|
|
113
|
+
// alone kept none, and the note said "check 0 or fewer purls per call". Now, in order:
|
|
114
|
+
// 1. SBOM_ROW_THIN, every row, every match (as before);
|
|
115
|
+
// 2. the same, with every row's matches capped at one number m, the most that fit: a row with
|
|
116
|
+
// more than m keeps its first m (the API's order: highest effective_score first) and carries
|
|
117
|
+
// matches_truncated, its own total and returned, and the CVE IDs among all its matches that
|
|
118
|
+
// are KEV-listed (kev_listed_cve_ids) or used in ransomware (ransomware_cve_ids), and the
|
|
119
|
+
// distinct fixed_in versions they name (fixed_in_versions), so what a triage ranks by first
|
|
120
|
+
// is still said of the matches cut; rows with m or fewer matches are untouched;
|
|
121
|
+
// 3. SBOM_ROW_COMPACT, each match cut to cve_id, kev_listed, ransomware, fixed_in and
|
|
122
|
+
// effective_score, the row keeping matched_package, matched_via and an upstream row's
|
|
123
|
+
// matched_version_from and purl_name, matches capped at one number m as in 2 (any m, 0
|
|
124
|
+
// included), with the same matches_truncated;
|
|
125
|
+
// 4. SBOM_ROW_MIN, the text's leanest level (index, purl, verdict, not_assessed_reason and
|
|
126
|
+
// cve_ids), with that matches_truncated on each row that had matches;
|
|
127
|
+
// 5. SBOM_ROW_MIN with matches_truncated cut to total and returned;
|
|
128
|
+
// and only then do rows leave (clean, then not assessed, then the last of the rest).
|
|
129
|
+
// #3132's review: round 1 had no level 3, and level 2 was taken at m = 0. The production
|
|
130
|
+
// synthetic's 201-purl probe (synthetic/probes.mjs SBOM_PROBE_PURLS, recorded on hosted 3.0.0 with
|
|
131
|
+
// full: true, 2026-10-11: 201 affected rows, 415 matches, 710,724 characters) then came back at
|
|
132
|
+
// level 5 with 132 rows, every match, KEV flag and fixed_in gone, and the probe's EXPECT (log4j-core
|
|
133
|
+
// 2.14.1's CVE-2021-44228 match, fixed_in 2.15.0; the upstream row's matched_via) failed, where
|
|
134
|
+
// 3.0.0 kept 43 rows thinned and passed. So now:
|
|
135
|
+
// - level 2 is taken only at m >= 1 (least): at m = 0 it holds less than level 3;
|
|
136
|
+
// - when not even level 5 keeps every row, rows leave at level 3 with matches capped at
|
|
137
|
+
// SBOM_LEAVING_CAP (leaveAt), not at level 5: a page of the size the note names then comes back
|
|
138
|
+
// with each row's KEV flags and fixed_in, which at level 5 no page size did;
|
|
139
|
+
// - once the cut is settled, the rows kept take back detail in input order while the result fits
|
|
140
|
+
// (giveBack): each the richest level above its own that fits, its matches at the compact level
|
|
141
|
+
// at the most that fit, stopping at the first row that takes back none.
|
|
142
|
+
// Measured on that recording (test/check_sbom-large-rows.test.mjs): 54 of 201 rows, each compact
|
|
143
|
+
// with every match (none has more than 7), the log4j-core row with its CVE-2021-44228 match and the
|
|
144
|
+
// upstream row with matched_via, at 29,959 characters.
|
|
145
|
+
// cve_ids is what the API caps at 200 a row, about 3,400 characters, so every row survives level 5
|
|
146
|
+
// unless the rows are many (the probe's 201, Juice Shop's full 840 purls).
|
|
147
|
+
export const SBOM_ROW_COMPACT = [
|
|
148
|
+
"index",
|
|
149
|
+
"purl",
|
|
150
|
+
"verdict",
|
|
151
|
+
"not_assessed_reason",
|
|
152
|
+
"cve_ids",
|
|
153
|
+
"matched_package",
|
|
154
|
+
"matched_via",
|
|
155
|
+
"matched_version_from",
|
|
156
|
+
"purl_name",
|
|
157
|
+
["matches", ["cve_id", "kev_listed", "ransomware", "fixed_in", "effective_score"]],
|
|
158
|
+
];
|
|
159
|
+
export const SBOM_ROW_MIN = ["index", "purl", "verdict", "not_assessed_reason", "cve_ids"];
|
|
160
|
+
// The most matches a row keeps, compact, when rows leave (the API's order, highest effective_score
|
|
161
|
+
// first); the rest are in its cve_ids and matches_truncated.
|
|
162
|
+
export const SBOM_LEAVING_CAP = 10;
|
|
163
|
+
// matches_truncated for a row whose matches data keeps `kept` of (none at levels 3 and 4); with
|
|
164
|
+
// `summary`, the three lists drawn from all of the row's matches. Absent on a row with no match
|
|
165
|
+
// cut, and on one with no matches.
|
|
166
|
+
function matchesTruncated(row, kept, summary) {
|
|
167
|
+
const ms = Array.isArray(row.matches) ? row.matches.filter(isObj) : [];
|
|
168
|
+
if (ms.length <= kept)
|
|
169
|
+
return {};
|
|
170
|
+
const ids = (f) => [...new Set(ms.filter((m) => m[f] === true && typeof m.cve_id === "string").map((m) => m.cve_id))];
|
|
171
|
+
const fixed = [...new Set(ms.map((m) => m.fixed_in).filter((v) => typeof v === "string"))];
|
|
172
|
+
return {
|
|
173
|
+
matches_truncated: {
|
|
174
|
+
total: ms.length,
|
|
175
|
+
returned: Math.min(kept, ms.length),
|
|
176
|
+
...(summary ? { kev_listed_cve_ids: ids("kev_listed"), ransomware_cve_ids: ids("ransomware"), fixed_in_versions: fixed } : {}),
|
|
177
|
+
},
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
const MATCHES_TRUNCATED_SAID = "carrying matches_truncated, which gives its own total and returned, and, of all its matches, the CVE IDs KEV-listed (kev_listed_cve_ids) or used in ransomware (ransomware_cve_ids) and the distinct fixed_in versions (fixed_in_versions)";
|
|
181
|
+
const cappedSaid = (cap) => `each row's matches keeping at most its first ${cap} (the API's order, highest effective_score first), a row with more than that ${MATCHES_TRUNCATED_SAID}`;
|
|
182
|
+
export const SBOM_ROW_LEVELS = [
|
|
183
|
+
SBOM_ROW_THIN,
|
|
184
|
+
{ keep: SBOM_ROW_THIN, capped: "matches", least: 1, mark: (row, cap) => matchesTruncated(row, cap, true), said: cappedSaid },
|
|
185
|
+
{ keep: SBOM_ROW_COMPACT, capped: "matches", mark: (row, cap) => matchesTruncated(row, cap, true), said: cappedSaid },
|
|
186
|
+
{ keep: SBOM_ROW_MIN, mark: (row) => matchesTruncated(row, 0, true), said: () => `a row that had matches ${MATCHES_TRUNCATED_SAID}` },
|
|
187
|
+
{ keep: SBOM_ROW_MIN, mark: (row) => matchesTruncated(row, 0, false), said: () => "a row that had matches carrying matches_truncated with its total and returned 0" },
|
|
188
|
+
];
|
|
189
|
+
const SBOM_LEAVE_AT = { level: 2, cap: SBOM_LEAVING_CAP };
|
|
190
|
+
// The page sentence (#3132): never "0 or fewer". Rows leave only once no level keeps them all, so a
|
|
191
|
+
// call of as many purls as data kept rows keeps each of its rows; when not one row fits, no page
|
|
192
|
+
// size does, and the note says where a row is read instead.
|
|
193
|
+
// #3132's review: the lists beside the rows go first, scan_manifest's not_checked before
|
|
194
|
+
// not_sent_purls, which keeps every purl or none (whole): those purls are the list a client
|
|
195
|
+
// retries, and round 1 kept 3 of the rate-limited case's 800. With none kept, the note gives them
|
|
196
|
+
// exactly as positions in the input (sendingSentences: coverage.sent + 1 to
|
|
197
|
+
// coverage.distinct_purls), and the room the rows leave goes back to not_sent_purls before
|
|
198
|
+
// not_checked.
|
|
105
199
|
export function sbomData(sides, page) {
|
|
200
|
+
const others = sides.filter((x) => x !== "not_sent_purls");
|
|
106
201
|
return {
|
|
107
202
|
steps: [
|
|
108
|
-
{ lists:
|
|
109
|
-
|
|
203
|
+
...(others.length ? [{ lists: others }] : []),
|
|
204
|
+
...(sides.includes("not_sent_purls") ? [{ lists: ["not_sent_purls"], whole: true }] : []),
|
|
205
|
+
{
|
|
206
|
+
lists: ["results"],
|
|
207
|
+
thin: SBOM_ROW_LEVELS,
|
|
208
|
+
leaveOutFirst: { field: "verdict", values: ["not_affected", "not_assessed"], why: "data.summary counts every verdict" },
|
|
209
|
+
leaveAt: SBOM_LEAVE_AT,
|
|
210
|
+
giveBack: true,
|
|
211
|
+
},
|
|
110
212
|
],
|
|
111
|
-
page: (t) =>
|
|
213
|
+
page: (t) => t.results && t.results.returned < t.results.total
|
|
214
|
+
? t.results.returned > 0
|
|
215
|
+
? page(t.results.returned)
|
|
216
|
+
: `Not one row fits data even cut to ${SBOM_ROW_COMPACT.map((x) => (typeof x === "string" ? x : x[0])).join(", ")}, so no number of purls per call does: the same call with full: true returns every row whole.`
|
|
217
|
+
: "",
|
|
112
218
|
};
|
|
113
219
|
}
|
|
114
220
|
const SBOM_DATA = sbomData(["not_sent_purls"], (k) => `To read every row in data, check ${k} or fewer purls per call.`);
|
|
@@ -288,11 +394,15 @@ export function checkSbomOutput(envelopeSchema) {
|
|
|
288
394
|
freshness: null,
|
|
289
395
|
});
|
|
290
396
|
}
|
|
397
|
+
// One output schema per process, not per request (3.0.2; index.ts envelopeSchema says why).
|
|
398
|
+
const OUTPUTS = new WeakMap();
|
|
291
399
|
// A literal server.registerTool(name, { title, description, … }) call with constant strings:
|
|
292
400
|
// marketing-site lib/mcpToolClaims.test.ts reads it here (it follows index.ts's call to this
|
|
293
401
|
// function) and holds the /pulse/mcp row to this title and description.
|
|
294
402
|
export function registerCheckSbom(server, deps) {
|
|
295
|
-
const
|
|
403
|
+
const envelope = deps.envelopeSchema;
|
|
404
|
+
const output = OUTPUTS.get(envelope) ?? checkSbomOutput(envelope);
|
|
405
|
+
OUTPUTS.set(envelope, output);
|
|
296
406
|
server.registerTool(CHECK_SBOM, {
|
|
297
407
|
title: CHECK_SBOM_TITLE,
|
|
298
408
|
description: CHECK_SBOM_DESCRIPTION,
|