@graphorin/eslint-plugin 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +12 -4
- package/dist/index.d.ts +28 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +235 -22
- package/dist/index.js.map +1 -1
- package/package.json +6 -3
- package/src/index.ts +113 -0
- package/src/rules/_comment-utils.ts +37 -0
- package/src/rules/no-bare-tool-exec.ts +120 -0
- package/src/rules/no-implicit-network-call.ts +236 -0
- package/src/rules/no-secret-in-deps.ts +113 -0
- package/src/rules/no-secret-unwrap.ts +118 -0
- package/src/rules/no-third-party-workflow-aliases.ts +100 -0
- package/src/rules/provider-middleware-order.ts +113 -0
- package/src/rules/tool-description-required.ts +75 -0
- package/src/rules/tool-examples-recommended.ts +61 -0
- package/src/rules/tool-parameter-naming.ts +75 -0
- package/src/tool-discovery.ts +804 -0
|
@@ -0,0 +1,804 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Static `tool({...})` discovery + per-tool grader. Used by both the
|
|
3
|
+
* three `tool-*` ESLint rules and by the `graphorin tools lint` CLI
|
|
4
|
+
* subcommand (Phase 15) so the rule logic has a single source of
|
|
5
|
+
* truth (per the working plan acceptance criteria for RB-49).
|
|
6
|
+
*
|
|
7
|
+
* The discovery is intentionally text-based - it scans a source
|
|
8
|
+
* string for `tool(` call expressions and extracts the immediate
|
|
9
|
+
* object literal that follows. The extractor handles the common
|
|
10
|
+
* formatting cases (single-line, multi-line, nested object/array
|
|
11
|
+
* literals, single + double-quoted strings, template literals) and
|
|
12
|
+
* gracefully skips invocations whose argument shape it cannot
|
|
13
|
+
* parse statically. The trade-off is documented: a project that
|
|
14
|
+
* hides its `tool({...})` calls behind a builder helper or a
|
|
15
|
+
* dynamic import will not be picked up by this lint surface; that
|
|
16
|
+
* is the documented contract for the v0.1 lint surface.
|
|
17
|
+
*
|
|
18
|
+
* **Comment-awareness (W-044).** Discovery AND grading run over a
|
|
19
|
+
* comment-blanked view of the source: line-comment and block-comment
|
|
20
|
+
* content is replaced with spaces (newlines preserved, so line numbers
|
|
21
|
+
* and offsets never shift) while string/template literals - and,
|
|
22
|
+
* conservatively, regex literals - are left untouched. A
|
|
23
|
+
* commented-out `tool({...})` is therefore never discovered, and a
|
|
24
|
+
* commented-out property (or a commented email inside a live
|
|
25
|
+
* `examples:` block) never feeds the grader. The ORIGINAL slice stays
|
|
26
|
+
* available as {@link DiscoveredTool.source} for reports; grading
|
|
27
|
+
* paths consume {@link DiscoveredTool.gradingSource}.
|
|
28
|
+
*
|
|
29
|
+
* **False-positive contract.** The scanner matches ANY callee whose
|
|
30
|
+
* last lexical token is `tool(` - including method calls like
|
|
31
|
+
* `.tool(` and locally-defined helpers that happen to share the name.
|
|
32
|
+
* Renamed or wrapped invocations (`const t = tool; t({...})`) are NOT
|
|
33
|
+
* seen. This is the accepted cost of the text-based surface that lets
|
|
34
|
+
* the CLI and the ESLint rules share one implementation (RB-49).
|
|
35
|
+
*
|
|
36
|
+
* **Per-tool grader rubric (RB-49 calibration):**
|
|
37
|
+
*
|
|
38
|
+
* - **description axis (0..40 points):**
|
|
39
|
+
* - 0 if missing / placeholder / shorter than 20 chars.
|
|
40
|
+
* - 16 if length >= 20.
|
|
41
|
+
* - 24 if length >= 30.
|
|
42
|
+
* - 32 if length >= 50.
|
|
43
|
+
* - 40 if length >= 80.
|
|
44
|
+
* - **examples axis (0..30 points):**
|
|
45
|
+
* - 0 if no examples or more than 5 (the documented upper bound).
|
|
46
|
+
* - 12 base for 1 example, +6 per additional, capped at 30.
|
|
47
|
+
* - -6 per PII finding (cap at 0).
|
|
48
|
+
* - **parameter naming axis (0..30 points):**
|
|
49
|
+
* - 30 base, deducted per finding.
|
|
50
|
+
* - -30/N per ambiguous-name finding (full penalty per param).
|
|
51
|
+
* - -10/N per numeric-suffix finding (partial penalty per param).
|
|
52
|
+
* - 15 baseline when no parameters are discoverable.
|
|
53
|
+
*
|
|
54
|
+
* Total: 0..100 points. Calibrated against the RB-49 fixture
|
|
55
|
+
* catalog so `wellDescribedTool` scores 82, `placeholderDescriptionTool`
|
|
56
|
+
* scores 20, and `examplesPiiTool` scores 61.
|
|
57
|
+
*
|
|
58
|
+
* @stable
|
|
59
|
+
*/
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* @stable
|
|
63
|
+
*/
|
|
64
|
+
export interface DiscoveredTool {
|
|
65
|
+
/** Source file the call was found in. */
|
|
66
|
+
readonly file: string;
|
|
67
|
+
/** 1-indexed line of the `tool(` token. */
|
|
68
|
+
readonly line: number;
|
|
69
|
+
/** Tool name extracted from the `name:` property when present. */
|
|
70
|
+
readonly name: string;
|
|
71
|
+
/** Tool description (`description:` value) when extractable. */
|
|
72
|
+
readonly description?: string;
|
|
73
|
+
/** Number of examples declared in the `examples:` array. */
|
|
74
|
+
readonly examplesCount: number;
|
|
75
|
+
/** Whether `examples:` is a non-empty array literal. */
|
|
76
|
+
readonly hasExamples: boolean;
|
|
77
|
+
/** Snapshot of identifiers referenced from the `inputSchema` Zod chain. */
|
|
78
|
+
readonly parameterNames: ReadonlyArray<string>;
|
|
79
|
+
/** Tags declared on the call (best-effort). */
|
|
80
|
+
readonly tags: ReadonlyArray<string>;
|
|
81
|
+
/**
|
|
82
|
+
* Raw object-literal source (ORIGINAL text, comments included).
|
|
83
|
+
* Useful for tests + as a context blob when the CLI needs to
|
|
84
|
+
* surface the original source in a report.
|
|
85
|
+
*/
|
|
86
|
+
readonly source: string;
|
|
87
|
+
/**
|
|
88
|
+
* W-044: the same slice with comments blanked - what discovery
|
|
89
|
+
* parsed and what every grading path (examples PII scan,
|
|
90
|
+
* description/parameter scoring) consumes. Same length and line
|
|
91
|
+
* structure as `source`.
|
|
92
|
+
*/
|
|
93
|
+
readonly gradingSource: string;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* @stable
|
|
98
|
+
*/
|
|
99
|
+
export type LintFindingKind =
|
|
100
|
+
| 'description-missing'
|
|
101
|
+
| 'description-too-short'
|
|
102
|
+
| 'description-placeholder'
|
|
103
|
+
| 'examples-missing'
|
|
104
|
+
| 'examples-too-many'
|
|
105
|
+
| 'examples-pii-detected'
|
|
106
|
+
| 'parameter-ambiguous'
|
|
107
|
+
| 'parameter-numeric-suffix';
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* @stable
|
|
111
|
+
*/
|
|
112
|
+
export interface LintFinding {
|
|
113
|
+
readonly rule:
|
|
114
|
+
| 'graphorin/tool-description-required'
|
|
115
|
+
| 'graphorin/tool-examples-recommended'
|
|
116
|
+
| 'graphorin/tool-parameter-naming';
|
|
117
|
+
readonly kind: LintFindingKind;
|
|
118
|
+
readonly severity: 'error' | 'warn' | 'info';
|
|
119
|
+
readonly message: string;
|
|
120
|
+
readonly toolName: string;
|
|
121
|
+
readonly file: string;
|
|
122
|
+
readonly line: number;
|
|
123
|
+
readonly hint?: string;
|
|
124
|
+
/**
|
|
125
|
+
* Optional matched-pattern context. Populated by the
|
|
126
|
+
* `examples-pii-detected` finding so reports can highlight which
|
|
127
|
+
* example payload triggered the rule.
|
|
128
|
+
*/
|
|
129
|
+
readonly matchedPattern?: string;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* @stable
|
|
134
|
+
*/
|
|
135
|
+
export interface ToolGraderScore {
|
|
136
|
+
readonly toolName: string;
|
|
137
|
+
readonly file: string;
|
|
138
|
+
readonly line: number;
|
|
139
|
+
readonly score: number;
|
|
140
|
+
readonly axes: {
|
|
141
|
+
readonly description: number;
|
|
142
|
+
readonly examples: number;
|
|
143
|
+
readonly parameterNaming: number;
|
|
144
|
+
};
|
|
145
|
+
readonly findings: ReadonlyArray<LintFinding>;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Generic identifiers the parameter-naming rule flags as ambiguous.
|
|
150
|
+
* Tools whose `inputSchema` references only specific identifiers
|
|
151
|
+
* (e.g. `userId`, `recipientEmail`, `apiKey`) get full credit on
|
|
152
|
+
* the naming axis.
|
|
153
|
+
*
|
|
154
|
+
* @stable
|
|
155
|
+
*/
|
|
156
|
+
export const AMBIGUOUS_PARAMETER_NAMES: ReadonlyArray<string> = Object.freeze([
|
|
157
|
+
'user',
|
|
158
|
+
'id',
|
|
159
|
+
'name',
|
|
160
|
+
'value',
|
|
161
|
+
'data',
|
|
162
|
+
'input',
|
|
163
|
+
'output',
|
|
164
|
+
'result',
|
|
165
|
+
'to',
|
|
166
|
+
'from',
|
|
167
|
+
'key',
|
|
168
|
+
'field',
|
|
169
|
+
]);
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Tag values that, when present in a tool's `tags: [...]` literal,
|
|
173
|
+
* suppress the parameter-naming rule for that tool. The opt-out
|
|
174
|
+
* exists so operators can defer the rename for a long tail of
|
|
175
|
+
* pre-RB-49 tools while the framework migrates without breaking
|
|
176
|
+
* calling code.
|
|
177
|
+
*
|
|
178
|
+
* @stable
|
|
179
|
+
*/
|
|
180
|
+
export const PARAMETER_NAMING_OPT_OUT_TAGS: ReadonlyArray<string> = Object.freeze([
|
|
181
|
+
'experimental',
|
|
182
|
+
'legacy',
|
|
183
|
+
]);
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Placeholder values the description-required rule treats as
|
|
187
|
+
* non-descriptions.
|
|
188
|
+
*
|
|
189
|
+
* @stable
|
|
190
|
+
*/
|
|
191
|
+
export const PLACEHOLDER_DESCRIPTIONS: ReadonlyArray<string> = Object.freeze([
|
|
192
|
+
'todo',
|
|
193
|
+
'fixme',
|
|
194
|
+
'tbd',
|
|
195
|
+
'description',
|
|
196
|
+
'placeholder',
|
|
197
|
+
]);
|
|
198
|
+
|
|
199
|
+
const MIN_DESCRIPTION_LENGTH = 20;
|
|
200
|
+
const MAX_EXAMPLES = 5;
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Discover every `tool({...})` invocation in a source string. The
|
|
204
|
+
* returned findings are stable + frozen so callers can pass them
|
|
205
|
+
* straight into a JSON report.
|
|
206
|
+
*
|
|
207
|
+
* @stable
|
|
208
|
+
*/
|
|
209
|
+
export function discoverToolCallsInSource(file: string, source: string): DiscoveredTool[] {
|
|
210
|
+
const out: DiscoveredTool[] = [];
|
|
211
|
+
// W-044: scan and parse over the comment-blanked view - a
|
|
212
|
+
// commented-out `tool({...})` is invisible, and commented braces or
|
|
213
|
+
// quotes can no longer derail the brace matcher. Offsets are shared
|
|
214
|
+
// with the original source (blanking preserves length + newlines).
|
|
215
|
+
// The REGEX additionally searches a strings-blanked view so a
|
|
216
|
+
// `tool(` inside a string/template literal never matches; literals
|
|
217
|
+
// are still parsed from the strings-intact view.
|
|
218
|
+
const blanked = blankComments(source);
|
|
219
|
+
const searchable = blankStringContents(blanked);
|
|
220
|
+
const regex = /\btool\s*\(\s*\{/g;
|
|
221
|
+
let match: RegExpExecArray | null = regex.exec(searchable);
|
|
222
|
+
while (match !== null) {
|
|
223
|
+
const objectStart = match.index + match[0].length - 1;
|
|
224
|
+
const objectEnd = matchBrace(blanked, objectStart);
|
|
225
|
+
if (objectEnd > 0) {
|
|
226
|
+
const literal = source.slice(objectStart, objectEnd + 1);
|
|
227
|
+
const gradingLiteral = blanked.slice(objectStart, objectEnd + 1);
|
|
228
|
+
const line = countLines(source, match.index);
|
|
229
|
+
const tool = parseToolLiteral(file, line, literal, gradingLiteral);
|
|
230
|
+
if (tool !== null) out.push(tool);
|
|
231
|
+
}
|
|
232
|
+
regex.lastIndex = objectEnd + 1;
|
|
233
|
+
match = regex.exec(searchable);
|
|
234
|
+
}
|
|
235
|
+
return out;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* W-044: blank the CONTENTS of string/template literals (quotes kept,
|
|
240
|
+
* newlines preserved) so the discovery regex cannot match `tool(`
|
|
241
|
+
* inside prose. Used only for SEARCHING - parsing reads the
|
|
242
|
+
* strings-intact view.
|
|
243
|
+
*/
|
|
244
|
+
function blankStringContents(source: string): string {
|
|
245
|
+
const out = source.split('');
|
|
246
|
+
let inString: '"' | "'" | '`' | null = null;
|
|
247
|
+
let i = 0;
|
|
248
|
+
while (i < source.length) {
|
|
249
|
+
const ch = source[i] as string;
|
|
250
|
+
if (inString !== null) {
|
|
251
|
+
if (ch === '\\') {
|
|
252
|
+
out[i] = ' ';
|
|
253
|
+
if (i + 1 < source.length && source[i + 1] !== '\n') out[i + 1] = ' ';
|
|
254
|
+
i += 2;
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
if (ch === inString) {
|
|
258
|
+
inString = null;
|
|
259
|
+
i += 1;
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
if (ch !== '\n') out[i] = ' ';
|
|
263
|
+
i += 1;
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
if (ch === '"' || ch === "'" || ch === '`') {
|
|
267
|
+
inString = ch as '"' | "'" | '`';
|
|
268
|
+
}
|
|
269
|
+
i += 1;
|
|
270
|
+
}
|
|
271
|
+
return out.join('');
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* W-044: replace line-comment and block-comment CONTENT with spaces
|
|
276
|
+
* while preserving every newline (offsets and 1-indexed lines never
|
|
277
|
+
* shift).
|
|
278
|
+
* String and template literals pass through untouched; regex literals
|
|
279
|
+
* are tracked with the classic prev-significant-token heuristic so a
|
|
280
|
+
* `/` inside one is never mistaken for a comment opener - when in
|
|
281
|
+
* doubt the scanner does NOT blank.
|
|
282
|
+
*
|
|
283
|
+
* @stable
|
|
284
|
+
*/
|
|
285
|
+
export function blankComments(source: string): string {
|
|
286
|
+
const out = source.split('');
|
|
287
|
+
let inString: '"' | "'" | '`' | null = null;
|
|
288
|
+
let prevSignificant = '';
|
|
289
|
+
let i = 0;
|
|
290
|
+
const isRegexStartContext = (): boolean =>
|
|
291
|
+
prevSignificant === '' || '([{=,:;!&|?+-*%<>~^'.includes(prevSignificant);
|
|
292
|
+
while (i < source.length) {
|
|
293
|
+
const ch = source[i] as string;
|
|
294
|
+
if (inString !== null) {
|
|
295
|
+
if (ch === '\\') {
|
|
296
|
+
i += 2;
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
if (ch === inString) inString = null;
|
|
300
|
+
i += 1;
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
if (ch === '"' || ch === "'" || ch === '`') {
|
|
304
|
+
inString = ch as '"' | "'" | '`';
|
|
305
|
+
prevSignificant = ch;
|
|
306
|
+
i += 1;
|
|
307
|
+
continue;
|
|
308
|
+
}
|
|
309
|
+
if (ch === '/' && source[i + 1] === '/') {
|
|
310
|
+
while (i < source.length && source[i] !== '\n') {
|
|
311
|
+
out[i] = ' ';
|
|
312
|
+
i += 1;
|
|
313
|
+
}
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
if (ch === '/' && source[i + 1] === '*') {
|
|
317
|
+
out[i] = ' ';
|
|
318
|
+
out[i + 1] = ' ';
|
|
319
|
+
i += 2;
|
|
320
|
+
while (i < source.length && !(source[i] === '*' && source[i + 1] === '/')) {
|
|
321
|
+
if (source[i] !== '\n') out[i] = ' ';
|
|
322
|
+
i += 1;
|
|
323
|
+
}
|
|
324
|
+
if (i < source.length) {
|
|
325
|
+
out[i] = ' ';
|
|
326
|
+
out[i + 1] = ' ';
|
|
327
|
+
i += 2;
|
|
328
|
+
}
|
|
329
|
+
continue;
|
|
330
|
+
}
|
|
331
|
+
if (ch === '/' && isRegexStartContext()) {
|
|
332
|
+
// Conservative regex-literal skip: consume to the closing
|
|
333
|
+
// unescaped '/', honouring character classes.
|
|
334
|
+
let j = i + 1;
|
|
335
|
+
let inClass = false;
|
|
336
|
+
while (j < source.length && source[j] !== '\n') {
|
|
337
|
+
const cj = source[j] as string;
|
|
338
|
+
if (cj === '\\') {
|
|
339
|
+
j += 2;
|
|
340
|
+
continue;
|
|
341
|
+
}
|
|
342
|
+
if (cj === '[') inClass = true;
|
|
343
|
+
else if (cj === ']') inClass = false;
|
|
344
|
+
else if (cj === '/' && !inClass) break;
|
|
345
|
+
j += 1;
|
|
346
|
+
}
|
|
347
|
+
if (j < source.length && source[j] === '/') {
|
|
348
|
+
prevSignificant = '/';
|
|
349
|
+
i = j + 1;
|
|
350
|
+
continue;
|
|
351
|
+
}
|
|
352
|
+
// No closing slash on the line - not a regex; fall through.
|
|
353
|
+
}
|
|
354
|
+
if (!/\s/.test(ch)) prevSignificant = ch;
|
|
355
|
+
i += 1;
|
|
356
|
+
}
|
|
357
|
+
return out.join('');
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/** Email PII pattern used by the examples-pii sub-check. */
|
|
361
|
+
const EMAIL_PATTERN = /[\w.+%-]+@[\w.-]+\.[A-Za-z]{2,}/;
|
|
362
|
+
|
|
363
|
+
/** Numeric-suffix pattern used by the parameter-naming sub-check. */
|
|
364
|
+
const NUMERIC_SUFFIX_PATTERN = /^[A-Za-z]+\d+$/;
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Run the three RB-49 rules against a discovered tool and return the
|
|
368
|
+
* findings. The CLI grader maps these findings into per-axis scores;
|
|
369
|
+
* the ESLint rules forward them to `context.report(...)`.
|
|
370
|
+
*
|
|
371
|
+
* @stable
|
|
372
|
+
*/
|
|
373
|
+
export function runToolRules(
|
|
374
|
+
tool: DiscoveredTool,
|
|
375
|
+
severityOverrides?: {
|
|
376
|
+
readonly toolDescription?: 'error' | 'warn' | 'off';
|
|
377
|
+
readonly toolExamples?: 'error' | 'warn' | 'off';
|
|
378
|
+
readonly toolParameterNaming?: 'error' | 'warn' | 'off';
|
|
379
|
+
},
|
|
380
|
+
): LintFinding[] {
|
|
381
|
+
const findings: LintFinding[] = [];
|
|
382
|
+
|
|
383
|
+
const descSeverity = severityOverrides?.toolDescription ?? 'error';
|
|
384
|
+
if (descSeverity !== 'off') {
|
|
385
|
+
const desc = tool.description?.trim() ?? '';
|
|
386
|
+
if (desc.length === 0) {
|
|
387
|
+
findings.push(
|
|
388
|
+
finding(
|
|
389
|
+
'graphorin/tool-description-required',
|
|
390
|
+
'description-missing',
|
|
391
|
+
descSeverity,
|
|
392
|
+
tool,
|
|
393
|
+
`tool '${tool.name}' has no description; add a description that explains what the tool does and when to use it.`,
|
|
394
|
+
),
|
|
395
|
+
);
|
|
396
|
+
} else if (PLACEHOLDER_DESCRIPTIONS.includes(desc.toLowerCase())) {
|
|
397
|
+
findings.push(
|
|
398
|
+
finding(
|
|
399
|
+
'graphorin/tool-description-required',
|
|
400
|
+
'description-placeholder',
|
|
401
|
+
descSeverity,
|
|
402
|
+
tool,
|
|
403
|
+
`tool '${tool.name}' description is a placeholder ('${tool.description}').`,
|
|
404
|
+
),
|
|
405
|
+
);
|
|
406
|
+
} else if (desc.length < MIN_DESCRIPTION_LENGTH) {
|
|
407
|
+
findings.push(
|
|
408
|
+
finding(
|
|
409
|
+
'graphorin/tool-description-required',
|
|
410
|
+
'description-too-short',
|
|
411
|
+
descSeverity,
|
|
412
|
+
tool,
|
|
413
|
+
`tool '${tool.name}' description is shorter than ${MIN_DESCRIPTION_LENGTH} characters.`,
|
|
414
|
+
),
|
|
415
|
+
);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
const examplesSeverity = severityOverrides?.toolExamples ?? 'warn';
|
|
420
|
+
if (examplesSeverity !== 'off') {
|
|
421
|
+
if (!tool.hasExamples || tool.examplesCount === 0) {
|
|
422
|
+
findings.push(
|
|
423
|
+
finding(
|
|
424
|
+
'graphorin/tool-examples-recommended',
|
|
425
|
+
'examples-missing',
|
|
426
|
+
examplesSeverity,
|
|
427
|
+
tool,
|
|
428
|
+
`tool '${tool.name}' has no examples; add 1-5 worked examples per Anthropic 2026 guidance.`,
|
|
429
|
+
),
|
|
430
|
+
);
|
|
431
|
+
} else if (tool.examplesCount > MAX_EXAMPLES) {
|
|
432
|
+
findings.push(
|
|
433
|
+
finding(
|
|
434
|
+
'graphorin/tool-examples-recommended',
|
|
435
|
+
'examples-too-many',
|
|
436
|
+
'error',
|
|
437
|
+
tool,
|
|
438
|
+
`tool '${tool.name}' declares ${tool.examplesCount} examples; the upper bound is ${MAX_EXAMPLES}.`,
|
|
439
|
+
),
|
|
440
|
+
);
|
|
441
|
+
}
|
|
442
|
+
// PII sub-check - fires once per matched email pattern in the
|
|
443
|
+
// examples block. Operators usually want synthetic data, not real
|
|
444
|
+
// addresses scraped from the corpus.
|
|
445
|
+
// W-044: grade over the blanked slice - a commented-out email
|
|
446
|
+
// inside a LIVE literal must not penalize the axis.
|
|
447
|
+
const emailMatch = EMAIL_PATTERN.exec(extractExamplesBlock(tool.gradingSource));
|
|
448
|
+
if (emailMatch !== null) {
|
|
449
|
+
const matched = emailMatch[0] as string;
|
|
450
|
+
findings.push(
|
|
451
|
+
finding(
|
|
452
|
+
'graphorin/tool-examples-recommended',
|
|
453
|
+
'examples-pii-detected',
|
|
454
|
+
'error',
|
|
455
|
+
tool,
|
|
456
|
+
`tool '${tool.name}' example contains a real-looking email '${matched}'; replace with synthetic data (e.g. 'user@example.com').`,
|
|
457
|
+
{
|
|
458
|
+
matchedPattern: matched,
|
|
459
|
+
hint: 'RB-49: examples must use synthetic test data so the rendered tool catalogue does not leak PII into provider context.',
|
|
460
|
+
},
|
|
461
|
+
),
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
const namingSeverity = severityOverrides?.toolParameterNaming ?? 'warn';
|
|
467
|
+
const namingOptedOut = tool.tags.some((t) => PARAMETER_NAMING_OPT_OUT_TAGS.includes(t));
|
|
468
|
+
if (namingSeverity !== 'off' && !namingOptedOut) {
|
|
469
|
+
for (const param of tool.parameterNames) {
|
|
470
|
+
if (AMBIGUOUS_PARAMETER_NAMES.includes(param)) {
|
|
471
|
+
findings.push(
|
|
472
|
+
finding(
|
|
473
|
+
'graphorin/tool-parameter-naming',
|
|
474
|
+
'parameter-ambiguous',
|
|
475
|
+
namingSeverity,
|
|
476
|
+
tool,
|
|
477
|
+
`tool '${tool.name}' uses ambiguous parameter name '${param}'; prefer a self-documenting name (e.g. '${param}Id', '${param}Email').`,
|
|
478
|
+
),
|
|
479
|
+
);
|
|
480
|
+
} else if (NUMERIC_SUFFIX_PATTERN.test(param)) {
|
|
481
|
+
findings.push(
|
|
482
|
+
finding(
|
|
483
|
+
'graphorin/tool-parameter-naming',
|
|
484
|
+
'parameter-numeric-suffix',
|
|
485
|
+
namingSeverity,
|
|
486
|
+
tool,
|
|
487
|
+
`tool '${tool.name}' uses numeric-suffix parameter name '${param}'; prefer a semantic name (e.g. 'queryText', 'userId').`,
|
|
488
|
+
),
|
|
489
|
+
);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
return findings;
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* Compute the per-tool grader score (0..100). Each axis is gated by
|
|
499
|
+
* the findings produced for that axis. The rubric is calibrated
|
|
500
|
+
* against the RB-49 fixture catalog (`wellDescribedTool` -> 82,
|
|
501
|
+
* `placeholderDescriptionTool` -> 20, `examplesPiiTool` -> 61).
|
|
502
|
+
*
|
|
503
|
+
* @stable
|
|
504
|
+
*/
|
|
505
|
+
export function gradeTool(
|
|
506
|
+
tool: DiscoveredTool,
|
|
507
|
+
findings: ReadonlyArray<LintFinding>,
|
|
508
|
+
): ToolGraderScore {
|
|
509
|
+
const descFindings = findings.filter((f) => f.rule === 'graphorin/tool-description-required');
|
|
510
|
+
const exampleFindings = findings.filter((f) => f.rule === 'graphorin/tool-examples-recommended');
|
|
511
|
+
const namingFindings = findings.filter((f) => f.rule === 'graphorin/tool-parameter-naming');
|
|
512
|
+
|
|
513
|
+
const description = scoreDescription(tool, descFindings);
|
|
514
|
+
const examples = scoreExamples(tool, exampleFindings);
|
|
515
|
+
const parameterNaming = scoreParameterNaming(tool, namingFindings);
|
|
516
|
+
|
|
517
|
+
return Object.freeze({
|
|
518
|
+
toolName: tool.name,
|
|
519
|
+
file: tool.file,
|
|
520
|
+
line: tool.line,
|
|
521
|
+
score: description + examples + parameterNaming,
|
|
522
|
+
axes: Object.freeze({ description, examples, parameterNaming }),
|
|
523
|
+
findings,
|
|
524
|
+
});
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
function scoreDescription(tool: DiscoveredTool, findings: ReadonlyArray<LintFinding>): number {
|
|
528
|
+
if (findings.length > 0) return 0;
|
|
529
|
+
const desc = tool.description?.trim() ?? '';
|
|
530
|
+
let lengthScore = 0;
|
|
531
|
+
if (desc.length >= 80) lengthScore = 40;
|
|
532
|
+
else if (desc.length >= 50) lengthScore = 32;
|
|
533
|
+
else if (desc.length >= 30) lengthScore = 24;
|
|
534
|
+
else if (desc.length >= MIN_DESCRIPTION_LENGTH) lengthScore = 16;
|
|
535
|
+
// W-044 anti-degenerate guard: 80 chars of repeated lorem must not
|
|
536
|
+
// score like real prose. Deterministic and deliberately narrow -
|
|
537
|
+
// real descriptions have at least 4 distinct words and no single
|
|
538
|
+
// word carrying more than half the text - so the RB-49 calibration
|
|
539
|
+
// fixtures are untouched. Degenerate text caps at the lowest
|
|
540
|
+
// non-zero tier (16).
|
|
541
|
+
if (lengthScore > 16 && desc.length > 0) {
|
|
542
|
+
const words = desc
|
|
543
|
+
.toLowerCase()
|
|
544
|
+
.split(/\s+/)
|
|
545
|
+
.filter((w) => w.length > 0);
|
|
546
|
+
const counts = new Map<string, number>();
|
|
547
|
+
for (const w of words) counts.set(w, (counts.get(w) ?? 0) + 1);
|
|
548
|
+
const unique = counts.size;
|
|
549
|
+
const topShare = words.length === 0 ? 0 : Math.max(...counts.values()) / words.length;
|
|
550
|
+
if (unique < 4 || topShare > 0.5) return 16;
|
|
551
|
+
}
|
|
552
|
+
return lengthScore;
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
function scoreExamples(tool: DiscoveredTool, findings: ReadonlyArray<LintFinding>): number {
|
|
556
|
+
if (tool.examplesCount === 0) return 0;
|
|
557
|
+
if (tool.examplesCount > MAX_EXAMPLES) return 0;
|
|
558
|
+
// 12 base for the first example + 6 per additional, cap at 30. This
|
|
559
|
+
// calibration produces wellDescribedTool == 12 (1 example) and
|
|
560
|
+
// placeholderDescriptionTool == 0 (no examples). Each PII finding
|
|
561
|
+
// subtracts 6 from the axis (cap at 0) so examplesPiiTool ends up
|
|
562
|
+
// with 6 (12 base - 6 PII penalty).
|
|
563
|
+
const base = Math.min(30, 12 + (tool.examplesCount - 1) * 6);
|
|
564
|
+
const piiCount = findings.filter((f) => f.kind === 'examples-pii-detected').length;
|
|
565
|
+
return Math.max(0, base - piiCount * 6);
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
function scoreParameterNaming(tool: DiscoveredTool, findings: ReadonlyArray<LintFinding>): number {
|
|
569
|
+
if (tool.parameterNames.length === 0) {
|
|
570
|
+
// No discoverable params - neither a positive nor a negative
|
|
571
|
+
// signal. Award the median.
|
|
572
|
+
return 15;
|
|
573
|
+
}
|
|
574
|
+
const total = tool.parameterNames.length;
|
|
575
|
+
const ambiguousCount = findings.filter((f) => f.kind === 'parameter-ambiguous').length;
|
|
576
|
+
const numericCount = findings.filter((f) => f.kind === 'parameter-numeric-suffix').length;
|
|
577
|
+
// Full penalty for an ambiguous-name finding (`user`, `id`, `to`, …):
|
|
578
|
+
// -30/N. Partial penalty for a numeric-suffix finding (`arg1`,
|
|
579
|
+
// `param2`, …): -10/N. Calibrated so a single-param `arg1` scores
|
|
580
|
+
// 20 (placeholderDescriptionTool) and a two-param `to`-+-`body`
|
|
581
|
+
// scores 15 (examplesPiiTool).
|
|
582
|
+
const score = 30 - (30 / total) * ambiguousCount - (10 / total) * numericCount;
|
|
583
|
+
return Math.max(0, Math.round(score));
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
function finding(
|
|
587
|
+
rule: LintFinding['rule'],
|
|
588
|
+
kind: LintFindingKind,
|
|
589
|
+
severity: 'error' | 'warn',
|
|
590
|
+
tool: DiscoveredTool,
|
|
591
|
+
message: string,
|
|
592
|
+
extra: { readonly hint?: string; readonly matchedPattern?: string } = {},
|
|
593
|
+
): LintFinding {
|
|
594
|
+
return Object.freeze({
|
|
595
|
+
rule,
|
|
596
|
+
kind,
|
|
597
|
+
severity,
|
|
598
|
+
message,
|
|
599
|
+
toolName: tool.name,
|
|
600
|
+
file: tool.file,
|
|
601
|
+
line: tool.line,
|
|
602
|
+
...(extra.hint !== undefined ? { hint: extra.hint } : {}),
|
|
603
|
+
...(extra.matchedPattern !== undefined ? { matchedPattern: extra.matchedPattern } : {}),
|
|
604
|
+
});
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/**
|
|
608
|
+
* Extract the substring inside the `examples: [...]` array literal so
|
|
609
|
+
* the PII detector can scan only the example payloads (not the rest of
|
|
610
|
+
* the tool body).
|
|
611
|
+
*
|
|
612
|
+
* @internal
|
|
613
|
+
*/
|
|
614
|
+
function extractExamplesBlock(literal: string): string {
|
|
615
|
+
const m = /examples\s*:\s*\[/.exec(literal);
|
|
616
|
+
if (m === null) return '';
|
|
617
|
+
const start = m.index + m[0].length - 1;
|
|
618
|
+
const end = matchBracket(literal, start);
|
|
619
|
+
if (end < 0) return '';
|
|
620
|
+
return literal.slice(start + 1, end);
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* Parse a string literal value from the immediate object literal.
|
|
625
|
+
* Supports `'`, `"`, and template strings (without interpolation).
|
|
626
|
+
*
|
|
627
|
+
* @internal
|
|
628
|
+
*/
|
|
629
|
+
function readStringProp(literal: string, prop: string): string | undefined {
|
|
630
|
+
const escapedProp = prop.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
631
|
+
const re = new RegExp(`(?:^|[\\s,{])${escapedProp}\\s*:\\s*(['"\`])`, 'm');
|
|
632
|
+
const m = re.exec(literal);
|
|
633
|
+
if (m === null) return undefined;
|
|
634
|
+
const quote = m[1] as string;
|
|
635
|
+
const start = m.index + m[0].length;
|
|
636
|
+
let i = start;
|
|
637
|
+
let out = '';
|
|
638
|
+
while (i < literal.length) {
|
|
639
|
+
const ch = literal[i] as string;
|
|
640
|
+
if (ch === '\\' && i + 1 < literal.length) {
|
|
641
|
+
out += literal[i + 1];
|
|
642
|
+
i += 2;
|
|
643
|
+
continue;
|
|
644
|
+
}
|
|
645
|
+
if (ch === quote) return out;
|
|
646
|
+
out += ch;
|
|
647
|
+
i += 1;
|
|
648
|
+
}
|
|
649
|
+
return undefined;
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
/**
|
|
653
|
+
* Count entries in an array literal value of the form `prop: [...]`.
|
|
654
|
+
*
|
|
655
|
+
* @internal
|
|
656
|
+
*/
|
|
657
|
+
function countArrayProp(
|
|
658
|
+
literal: string,
|
|
659
|
+
prop: string,
|
|
660
|
+
): { readonly count: number; readonly present: boolean } {
|
|
661
|
+
const escapedProp = prop.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
662
|
+
const re = new RegExp(`(?:^|[\\s,{])${escapedProp}\\s*:\\s*\\[`, 'm');
|
|
663
|
+
const m = re.exec(literal);
|
|
664
|
+
if (m === null) return { count: 0, present: false };
|
|
665
|
+
const start = m.index + m[0].length - 1;
|
|
666
|
+
const end = matchBracket(literal, start);
|
|
667
|
+
if (end < 0) return { count: 0, present: true };
|
|
668
|
+
// Strip a trailing comma so it does not inflate the count.
|
|
669
|
+
const inner = literal
|
|
670
|
+
.slice(start + 1, end)
|
|
671
|
+
.trim()
|
|
672
|
+
.replace(/,\s*$/, '');
|
|
673
|
+
if (inner.length === 0) return { count: 0, present: true };
|
|
674
|
+
// Count top-level commas (depth + string aware).
|
|
675
|
+
let depth = 0;
|
|
676
|
+
let inString: '"' | "'" | '`' | null = null;
|
|
677
|
+
let count = 1;
|
|
678
|
+
for (let i = 0; i < inner.length; i += 1) {
|
|
679
|
+
const ch = inner[i] as string;
|
|
680
|
+
if (inString !== null) {
|
|
681
|
+
if (ch === '\\') {
|
|
682
|
+
i += 1;
|
|
683
|
+
continue;
|
|
684
|
+
}
|
|
685
|
+
if (ch === inString) inString = null;
|
|
686
|
+
continue;
|
|
687
|
+
}
|
|
688
|
+
if (ch === '"' || ch === "'" || ch === '`') {
|
|
689
|
+
inString = ch as '"' | "'" | '`';
|
|
690
|
+
continue;
|
|
691
|
+
}
|
|
692
|
+
if (ch === '(' || ch === '[' || ch === '{') depth += 1;
|
|
693
|
+
else if (ch === ')' || ch === ']' || ch === '}') depth -= 1;
|
|
694
|
+
else if (ch === ',' && depth === 0) count += 1;
|
|
695
|
+
}
|
|
696
|
+
return { count, present: true };
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
/**
|
|
700
|
+
* Extract identifiers passed into a `z.object({...})` schema as
|
|
701
|
+
* top-level keys.
|
|
702
|
+
*
|
|
703
|
+
* @internal
|
|
704
|
+
*/
|
|
705
|
+
function extractParameterNames(literal: string): string[] {
|
|
706
|
+
const m = /inputSchema\s*:\s*z\s*\.\s*object\s*\(\s*\{/.exec(literal);
|
|
707
|
+
if (m === null) return [];
|
|
708
|
+
const start = m.index + m[0].length - 1;
|
|
709
|
+
const end = matchBrace(literal, start);
|
|
710
|
+
if (end < 0) return [];
|
|
711
|
+
const inner = literal.slice(start + 1, end);
|
|
712
|
+
const names: string[] = [];
|
|
713
|
+
const idRegex = /(^|[\s,{])([A-Za-z_$][A-Za-z0-9_$]*)\s*:/g;
|
|
714
|
+
let id = idRegex.exec(inner);
|
|
715
|
+
while (id !== null) {
|
|
716
|
+
const name = id[2] as string;
|
|
717
|
+
if (!names.includes(name)) names.push(name);
|
|
718
|
+
id = idRegex.exec(inner);
|
|
719
|
+
}
|
|
720
|
+
return names;
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
function parseToolLiteral(
|
|
724
|
+
file: string,
|
|
725
|
+
line: number,
|
|
726
|
+
literal: string,
|
|
727
|
+
gradingLiteral: string,
|
|
728
|
+
): DiscoveredTool | null {
|
|
729
|
+
// W-044: every extraction parses the BLANKED slice so commented-out
|
|
730
|
+
// properties inside a live literal never count; string values are
|
|
731
|
+
// preserved verbatim by the blanker.
|
|
732
|
+
const name = readStringProp(gradingLiteral, 'name') ?? '<anonymous>';
|
|
733
|
+
const description = readStringProp(gradingLiteral, 'description');
|
|
734
|
+
const examples = countArrayProp(gradingLiteral, 'examples');
|
|
735
|
+
const tagsArr = countArrayProp(gradingLiteral, 'tags');
|
|
736
|
+
const tags: string[] = [];
|
|
737
|
+
if (tagsArr.present && tagsArr.count > 0) {
|
|
738
|
+
const m = /tags\s*:\s*\[([^\]]*)\]/.exec(gradingLiteral);
|
|
739
|
+
if (m !== null) {
|
|
740
|
+
const inner = (m[1] as string) ?? '';
|
|
741
|
+
const tagRegex = /['"`]([^'"`]+)['"`]/g;
|
|
742
|
+
let t = tagRegex.exec(inner);
|
|
743
|
+
while (t !== null) {
|
|
744
|
+
tags.push(t[1] as string);
|
|
745
|
+
t = tagRegex.exec(inner);
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
const parameterNames = extractParameterNames(gradingLiteral);
|
|
750
|
+
return Object.freeze({
|
|
751
|
+
file,
|
|
752
|
+
line,
|
|
753
|
+
name,
|
|
754
|
+
...(description !== undefined ? { description } : {}),
|
|
755
|
+
examplesCount: examples.count,
|
|
756
|
+
hasExamples: examples.present,
|
|
757
|
+
parameterNames: Object.freeze(parameterNames),
|
|
758
|
+
tags: Object.freeze(tags),
|
|
759
|
+
source: literal,
|
|
760
|
+
gradingSource: gradingLiteral,
|
|
761
|
+
});
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
function countLines(source: string, index: number): number {
|
|
765
|
+
let n = 1;
|
|
766
|
+
for (let i = 0; i < index; i += 1) {
|
|
767
|
+
if (source[i] === '\n') n += 1;
|
|
768
|
+
}
|
|
769
|
+
return n;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
function matchBrace(source: string, openIndex: number): number {
|
|
773
|
+
return matchPair(source, openIndex, '{', '}');
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
function matchBracket(source: string, openIndex: number): number {
|
|
777
|
+
return matchPair(source, openIndex, '[', ']');
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
function matchPair(source: string, openIndex: number, open: string, close: string): number {
|
|
781
|
+
let depth = 0;
|
|
782
|
+
let inString: '"' | "'" | '`' | null = null;
|
|
783
|
+
for (let i = openIndex; i < source.length; i += 1) {
|
|
784
|
+
const ch = source[i] as string;
|
|
785
|
+
if (inString !== null) {
|
|
786
|
+
if (ch === '\\') {
|
|
787
|
+
i += 1;
|
|
788
|
+
continue;
|
|
789
|
+
}
|
|
790
|
+
if (ch === inString) inString = null;
|
|
791
|
+
continue;
|
|
792
|
+
}
|
|
793
|
+
if (ch === '"' || ch === "'" || ch === '`') {
|
|
794
|
+
inString = ch as '"' | "'" | '`';
|
|
795
|
+
continue;
|
|
796
|
+
}
|
|
797
|
+
if (ch === open) depth += 1;
|
|
798
|
+
else if (ch === close) {
|
|
799
|
+
depth -= 1;
|
|
800
|
+
if (depth === 0) return i;
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
return -1;
|
|
804
|
+
}
|