mcp-context-cost 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +27 -0
- package/dist/cli.js +83 -19
- package/dist/core/server-diff.d.ts +16 -0
- package/dist/core/server-diff.js +36 -1
- package/dist/sweep/regressions.js +15 -0
- package/package.json +1 -1
package/dist/cli.d.ts
CHANGED
|
@@ -27,3 +27,30 @@ export declare function unknownFlags(argv: string[], spec: {
|
|
|
27
27
|
value: string[];
|
|
28
28
|
boolean: string[];
|
|
29
29
|
}): string[];
|
|
30
|
+
/**
|
|
31
|
+
* Every value a value-taking flag was given, in either accepted spelling:
|
|
32
|
+
* `--flag value` and `--flag=value`.
|
|
33
|
+
*
|
|
34
|
+
* Both forms are read here because reading only one of them is the same bug as
|
|
35
|
+
* ignoring an unknown flag. `--max-increase=100` was accepted by
|
|
36
|
+
* `unknownFlags` (which splits on `=`) and then invisible to a reader that only
|
|
37
|
+
* matched the bare token, so the gate it asked for silently did not run and the
|
|
38
|
+
* command exited 0 — a green check on a check that never happened.
|
|
39
|
+
*/
|
|
40
|
+
export declare function flagValues(argv: string[], name: string): string[];
|
|
41
|
+
/** The last value given for a flag, or undefined when the flag is absent. */
|
|
42
|
+
export declare function flagValue(argv: string[], name: string): string | undefined;
|
|
43
|
+
/**
|
|
44
|
+
* Value-taking flags that appear with no usable value.
|
|
45
|
+
*
|
|
46
|
+
* A flag present without its value is a *usage error*, never an absent flag.
|
|
47
|
+
* `--max-increase` as the last argument — what a CI template renders when its
|
|
48
|
+
* variable is empty — otherwise reads as "no gate was asked for", and the run
|
|
49
|
+
* exits 0 on a change that should have failed it. That is the same green-check
|
|
50
|
+
* failure `unknownFlags` exists to prevent, reached through a different door,
|
|
51
|
+
* so it is refused in the same place and with the same severity.
|
|
52
|
+
*/
|
|
53
|
+
export declare function valuelessFlags(argv: string[], spec: {
|
|
54
|
+
value: string[];
|
|
55
|
+
boolean: string[];
|
|
56
|
+
}): string[];
|
package/dist/cli.js
CHANGED
|
@@ -85,16 +85,87 @@ export function unknownFlags(argv, spec) {
|
|
|
85
85
|
}
|
|
86
86
|
return unknown;
|
|
87
87
|
}
|
|
88
|
+
/**
|
|
89
|
+
* Every value a value-taking flag was given, in either accepted spelling:
|
|
90
|
+
* `--flag value` and `--flag=value`.
|
|
91
|
+
*
|
|
92
|
+
* Both forms are read here because reading only one of them is the same bug as
|
|
93
|
+
* ignoring an unknown flag. `--max-increase=100` was accepted by
|
|
94
|
+
* `unknownFlags` (which splits on `=`) and then invisible to a reader that only
|
|
95
|
+
* matched the bare token, so the gate it asked for silently did not run and the
|
|
96
|
+
* command exited 0 — a green check on a check that never happened.
|
|
97
|
+
*/
|
|
98
|
+
export function flagValues(argv, name) {
|
|
99
|
+
const out = [];
|
|
100
|
+
for (let i = 0; i < argv.length; i++) {
|
|
101
|
+
const tok = argv[i];
|
|
102
|
+
if (tok === `--${name}`) {
|
|
103
|
+
const next = argv[i + 1];
|
|
104
|
+
// A following flag is not this flag's value; that case is a usage error,
|
|
105
|
+
// caught by `valuelessFlags`, and must not be read as a value here.
|
|
106
|
+
if (next !== undefined && !next.startsWith('--'))
|
|
107
|
+
out.push(next);
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (tok.startsWith(`--${name}=`))
|
|
111
|
+
out.push(tok.slice(name.length + 3));
|
|
112
|
+
}
|
|
113
|
+
return out;
|
|
114
|
+
}
|
|
115
|
+
/** The last value given for a flag, or undefined when the flag is absent. */
|
|
116
|
+
export function flagValue(argv, name) {
|
|
117
|
+
const values = flagValues(argv, name);
|
|
118
|
+
return values.length ? values[values.length - 1] : undefined;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Value-taking flags that appear with no usable value.
|
|
122
|
+
*
|
|
123
|
+
* A flag present without its value is a *usage error*, never an absent flag.
|
|
124
|
+
* `--max-increase` as the last argument — what a CI template renders when its
|
|
125
|
+
* variable is empty — otherwise reads as "no gate was asked for", and the run
|
|
126
|
+
* exits 0 on a change that should have failed it. That is the same green-check
|
|
127
|
+
* failure `unknownFlags` exists to prevent, reached through a different door,
|
|
128
|
+
* so it is refused in the same place and with the same severity.
|
|
129
|
+
*/
|
|
130
|
+
export function valuelessFlags(argv, spec) {
|
|
131
|
+
const bad = [];
|
|
132
|
+
for (let i = 0; i < argv.length; i++) {
|
|
133
|
+
const tok = argv[i];
|
|
134
|
+
if (!tok.startsWith('--'))
|
|
135
|
+
continue;
|
|
136
|
+
const name = tok.slice(2).split('=')[0];
|
|
137
|
+
if (!spec.value.includes(name))
|
|
138
|
+
continue;
|
|
139
|
+
if (tok.includes('=')) {
|
|
140
|
+
if (tok.slice(name.length + 3) === '')
|
|
141
|
+
bad.push(`--${name}`);
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
const next = argv[i + 1];
|
|
145
|
+
if (next === undefined || next.startsWith('--'))
|
|
146
|
+
bad.push(`--${name}`);
|
|
147
|
+
else
|
|
148
|
+
i++; // consume the value, so `--command "--weird"` is not re-read as a flag
|
|
149
|
+
}
|
|
150
|
+
return bad;
|
|
151
|
+
}
|
|
88
152
|
function rejectUnknownFlags(cmd, argv, spec) {
|
|
89
153
|
const bad = unknownFlags(argv, spec);
|
|
90
|
-
if (
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
154
|
+
if (bad.length) {
|
|
155
|
+
const all = [...spec.value, ...spec.boolean].sort().map((f) => `--${f}`).join(' ');
|
|
156
|
+
console.error(`unknown flag for \`${cmd}\`: ${bad.join(', ')}`);
|
|
157
|
+
console.error(`this is mcp-context-cost ${cliVersion()} — if you copied the command from the README,`);
|
|
158
|
+
console.error(`your install may be older than the docs. Try: npx -y mcp-context-cost@latest ${cmd} ...`);
|
|
159
|
+
console.error(`known flags for ${cmd}: ${all}`);
|
|
160
|
+
process.exit(2);
|
|
161
|
+
}
|
|
162
|
+
const empty = valuelessFlags(argv, spec);
|
|
163
|
+
if (empty.length) {
|
|
164
|
+
console.error(`flag with no value for \`${cmd}\`: ${empty.join(', ')}`);
|
|
165
|
+
console.error(`a flag given without its value is refused rather than ignored: ignoring it would run`);
|
|
166
|
+
console.error(`a command that quietly does less than it was asked to — a gate that never gates.`);
|
|
167
|
+
process.exit(2);
|
|
168
|
+
}
|
|
98
169
|
}
|
|
99
170
|
const [, , cmd, ...rest] = process.argv;
|
|
100
171
|
if (cmd === 'audit') {
|
|
@@ -113,11 +184,8 @@ if (cmd === 'audit') {
|
|
|
113
184
|
],
|
|
114
185
|
boolean: ['json', 'docker', 'claude', 'suggest', 'changed'],
|
|
115
186
|
});
|
|
116
|
-
const argOf = (name) =>
|
|
117
|
-
|
|
118
|
-
return i >= 0 ? rest[i + 1] : undefined;
|
|
119
|
-
};
|
|
120
|
-
const all = (name) => rest.flatMap((a, i) => (a === `--${name}` && rest[i + 1] ? [rest[i + 1]] : []));
|
|
187
|
+
const argOf = (name) => flagValue(rest, name);
|
|
188
|
+
const all = (name) => flagValues(rest, name);
|
|
121
189
|
const json = rest.includes('--json');
|
|
122
190
|
const numeric = (name) => {
|
|
123
191
|
const raw = argOf(name);
|
|
@@ -249,8 +317,7 @@ if (cmd === 'audit') {
|
|
|
249
317
|
else if (cmd === 'verify') {
|
|
250
318
|
rejectUnknownFlags('verify', rest, { value: ['remote'], boolean: ['json'] });
|
|
251
319
|
const json = rest.includes('--json');
|
|
252
|
-
const
|
|
253
|
-
const remoteUrl = remoteIdx >= 0 ? rest[remoteIdx + 1] : undefined;
|
|
320
|
+
const remoteUrl = flagValue(rest, 'remote');
|
|
254
321
|
const path = rest.find((a) => !a.startsWith('--') && a !== remoteUrl);
|
|
255
322
|
if (!remoteUrl && !path) {
|
|
256
323
|
console.error('usage: mcp-context-cost verify <measurement.json> [--json]');
|
|
@@ -298,10 +365,7 @@ else if (cmd === 'measure') {
|
|
|
298
365
|
value: ['name', 'command', 'remote', 'timeout', 'docker-image', 'baseline', 'max-increase', 'budget'],
|
|
299
366
|
boolean: ['docker'],
|
|
300
367
|
});
|
|
301
|
-
const argOf = (name) =>
|
|
302
|
-
const i = rest.indexOf(`--${name}`);
|
|
303
|
-
return i >= 0 ? rest[i + 1] : undefined;
|
|
304
|
-
};
|
|
368
|
+
const argOf = (name) => flagValue(rest, name);
|
|
305
369
|
const command = argOf('command');
|
|
306
370
|
const remoteUrl = argOf('remote');
|
|
307
371
|
if (remoteUrl && !/^https?:\/\//i.test(remoteUrl)) {
|
|
@@ -46,6 +46,22 @@ export interface ServerDiff {
|
|
|
46
46
|
/** Per-tool breakdown of an established change. */
|
|
47
47
|
attribution: ToolAttribution | null;
|
|
48
48
|
}
|
|
49
|
+
/**
|
|
50
|
+
* Whether two measurements describe the same server, as far as what they
|
|
51
|
+
* recorded can say.
|
|
52
|
+
*
|
|
53
|
+
* A baseline is a path the caller passes, and nothing about the path proves it
|
|
54
|
+
* belongs to the server being measured — a monorepo with `baseline-a.json` and
|
|
55
|
+
* `baseline-b.json` is one copy-paste away from diffing two unrelated servers
|
|
56
|
+
* and reporting the difference as though it meant something. Both sides record
|
|
57
|
+
* `serverName` (what the server called itself at `initialize`), so where both
|
|
58
|
+
* carry one and they disagree, the comparison is refused.
|
|
59
|
+
*
|
|
60
|
+
* Only a disagreement counts. A measurement predating the field, or one whose
|
|
61
|
+
* server reports no name, is unknown rather than mismatched, and unknown is not
|
|
62
|
+
* evidence of anything — the same rule the isolation column follows.
|
|
63
|
+
*/
|
|
64
|
+
export declare function sameServer(baseline: Measurement, current: Measurement): boolean;
|
|
49
65
|
export declare function diffServer(name: string, baseline: Measurement | null, current: Measurement): ServerDiff;
|
|
50
66
|
export interface ServerGate {
|
|
51
67
|
pass: boolean;
|
package/dist/core/server-diff.js
CHANGED
|
@@ -35,6 +35,28 @@ const sideOf = (m) => {
|
|
|
35
35
|
measuredAt: v.date,
|
|
36
36
|
};
|
|
37
37
|
};
|
|
38
|
+
/**
|
|
39
|
+
* Whether two measurements describe the same server, as far as what they
|
|
40
|
+
* recorded can say.
|
|
41
|
+
*
|
|
42
|
+
* A baseline is a path the caller passes, and nothing about the path proves it
|
|
43
|
+
* belongs to the server being measured — a monorepo with `baseline-a.json` and
|
|
44
|
+
* `baseline-b.json` is one copy-paste away from diffing two unrelated servers
|
|
45
|
+
* and reporting the difference as though it meant something. Both sides record
|
|
46
|
+
* `serverName` (what the server called itself at `initialize`), so where both
|
|
47
|
+
* carry one and they disagree, the comparison is refused.
|
|
48
|
+
*
|
|
49
|
+
* Only a disagreement counts. A measurement predating the field, or one whose
|
|
50
|
+
* server reports no name, is unknown rather than mismatched, and unknown is not
|
|
51
|
+
* evidence of anything — the same rule the isolation column follows.
|
|
52
|
+
*/
|
|
53
|
+
export function sameServer(baseline, current) {
|
|
54
|
+
const a = typeof baseline.serverName === 'string' ? baseline.serverName : '';
|
|
55
|
+
const b = typeof current.serverName === 'string' ? current.serverName : '';
|
|
56
|
+
if (!a || !b)
|
|
57
|
+
return true;
|
|
58
|
+
return a === b;
|
|
59
|
+
}
|
|
38
60
|
export function diffServer(name, baseline, current) {
|
|
39
61
|
const before = sideOf(baseline);
|
|
40
62
|
const after = sideOf(current);
|
|
@@ -65,6 +87,16 @@ export function diffServer(name, baseline, current) {
|
|
|
65
87
|
: 'no baseline to compare against',
|
|
66
88
|
};
|
|
67
89
|
}
|
|
90
|
+
// Checked only once both sides measured, so a mismatch is reported as what it
|
|
91
|
+
// is rather than hidden behind an unmeasured side.
|
|
92
|
+
if (!sameServer(baseline, current)) {
|
|
93
|
+
return {
|
|
94
|
+
...base,
|
|
95
|
+
problem: `the baseline measured '${baseline.serverName}' and this run measured ` +
|
|
96
|
+
`'${current.serverName}' — a difference between two servers is not a change to either, ` +
|
|
97
|
+
`so nothing is compared`,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
68
100
|
const identical = before.canonicalSha256 === after.canonicalSha256;
|
|
69
101
|
const beforeVec = vectorEntryOf(baseline);
|
|
70
102
|
const afterVec = vectorEntryOf(current);
|
|
@@ -101,11 +133,14 @@ export function evaluateServerGate(diff, limits) {
|
|
|
101
133
|
// A short reason rather than the diff's full sentence: the diff has
|
|
102
134
|
// already printed that above, and a gate line that repeats it in full
|
|
103
135
|
// buries the verdict it exists to state.
|
|
136
|
+
// Both sides measured and still not exact means exactly one thing today:
|
|
137
|
+
// the baseline describes a different server. Kept as its own branch so a
|
|
138
|
+
// future non-exact case cannot inherit this sentence by accident.
|
|
104
139
|
const reason = !diff.after
|
|
105
140
|
? 'this run produced no number'
|
|
106
141
|
: !diff.before
|
|
107
142
|
? 'the baseline carries no measured number'
|
|
108
|
-
: '
|
|
143
|
+
: 'the baseline describes a different server';
|
|
109
144
|
return {
|
|
110
145
|
pass: false,
|
|
111
146
|
failure: `INCREASE FAIL: ${reason}, so the change could not be established and the gate has not passed.`,
|
|
@@ -74,11 +74,24 @@ export function loadToolVectors(server, root = process.cwd()) {
|
|
|
74
74
|
export function writeCaptureIndex(entries, root = process.cwd()) {
|
|
75
75
|
const captures = {};
|
|
76
76
|
const current = {};
|
|
77
|
+
// A hash two servers share identifies neither of them. The set already holds
|
|
78
|
+
// near-duplicate pairs (`redis`/`redis-legacy`, `github`/`github-legacy`), and
|
|
79
|
+
// one package listed under two slugs would produce byte-identical captures —
|
|
80
|
+
// whereupon the later write would silently rename the earlier server's capture
|
|
81
|
+
// and `audit --changed` would print the wrong name with full confidence.
|
|
82
|
+
// Ambiguous hashes are dropped instead, so `identify` answers `unknown`: an
|
|
83
|
+
// absence of a record, which is true, rather than a confident misattribution.
|
|
84
|
+
const ambiguous = new Set();
|
|
77
85
|
for (const entry of entries) {
|
|
78
86
|
const vectors = loadToolVectors(entry.name, root);
|
|
79
87
|
if (!vectors || vectors.entries.length === 0)
|
|
80
88
|
continue;
|
|
81
89
|
for (const e of vectors.entries) {
|
|
90
|
+
const held = captures[e.canonicalSha256];
|
|
91
|
+
if (held && held.server !== entry.name) {
|
|
92
|
+
ambiguous.add(e.canonicalSha256);
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
82
95
|
captures[e.canonicalSha256] = {
|
|
83
96
|
server: entry.name,
|
|
84
97
|
date: e.date,
|
|
@@ -88,6 +101,8 @@ export function writeCaptureIndex(entries, root = process.cwd()) {
|
|
|
88
101
|
}
|
|
89
102
|
current[entry.name] = vectors.entries[vectors.entries.length - 1].canonicalSha256;
|
|
90
103
|
}
|
|
104
|
+
for (const sha of ambiguous)
|
|
105
|
+
delete captures[sha];
|
|
91
106
|
const index = {
|
|
92
107
|
method: CAPTURE_INDEX_METHOD,
|
|
93
108
|
generatedAt: new Date().toISOString().slice(0, 10),
|
package/package.json
CHANGED