@andrueandersoncs/lint-anything 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -42
- package/dist/cli.js +61 -61
- package/dist/lib.js +6 -58
- package/dist/types/lib.d.ts +2 -2
- package/dist/types/lint.d.ts +7 -23
- package/dist/types/policy.d.ts +1 -14
- package/package.json +1 -1
- package/skills/lint-anything/SKILL.md +7 -25
- package/skills/lint-anything/scripts/score.js +40 -40
package/dist/lib.js
CHANGED
|
@@ -400,37 +400,6 @@ var runSelector = Effect3.fn("runSelector")(function* (policy, cwd, runs) {
|
|
|
400
400
|
});
|
|
401
401
|
|
|
402
402
|
// src/lint.ts
|
|
403
|
-
var itemPlace = (finding, inFile) => finding.lines === undefined ? finding.item : `${inFile ? "" : `${finding.path}:`}${finding.lines[0]}-${finding.lines[1]}`;
|
|
404
|
-
var unjudgedCauses = ["request too large", "declined"];
|
|
405
|
-
var aggregate = (policy, limit, items) => {
|
|
406
|
-
const groups = new Map;
|
|
407
|
-
if (limit.per === "run")
|
|
408
|
-
groups.set(undefined, [...items]);
|
|
409
|
-
else
|
|
410
|
-
for (const item of items)
|
|
411
|
-
groups.set(item.path, [...groups.get(item.path) ?? [], item]);
|
|
412
|
-
return [...groups].map(([path, group]) => {
|
|
413
|
-
const violating = group.filter((item) => item.verdict === "violation");
|
|
414
|
-
const review = group.filter((item) => item.verdict === "review").length;
|
|
415
|
-
const unjudged = group.filter((item) => item.verdict === "inconclusive");
|
|
416
|
-
const verdict = violating.length > limit.max ? "violation" : violating.length + unjudged.length > limit.max ? "inconclusive" : "pass";
|
|
417
|
-
const places = violating.map((item) => itemPlace(item, path !== undefined)).join(", ");
|
|
418
|
-
return {
|
|
419
|
-
policy,
|
|
420
|
-
...path === undefined ? {} : { path },
|
|
421
|
-
verdict,
|
|
422
|
-
reason: [
|
|
423
|
-
`${violating.length} of ${group.length} items violate the rule (at most ${limit.max} allowed)${places === "" ? "" : `: ${places}`}`,
|
|
424
|
-
...review > 0 ? [`${review} more in review`] : [],
|
|
425
|
-
...unjudgedCauses.flatMap((cause) => {
|
|
426
|
-
const count = unjudged.filter((item) => (item.unjudged ?? "request too large") === cause).length;
|
|
427
|
-
return count > 0 ? [`${count} not judged (${cause})`] : [];
|
|
428
|
-
})
|
|
429
|
-
].join("; "),
|
|
430
|
-
violating: violating.map((item) => item.item)
|
|
431
|
-
};
|
|
432
|
-
});
|
|
433
|
-
};
|
|
434
403
|
var reviewFloor = 0.5;
|
|
435
404
|
var defaultThreshold = 0.65;
|
|
436
405
|
var verdictOf = (violates, floor, threshold) => violates <= floor ? "pass" : violates >= threshold ? "violation" : "review";
|
|
@@ -440,12 +409,7 @@ var toFinding = (violates, base, policy, threshold) => Result2.match(violates, {
|
|
|
440
409
|
const verdict = verdictOf(violates, reviewFloor, threshold);
|
|
441
410
|
return { ...base, verdict, violates, ...verdict === "pass" ? {} : { reason: policy.criteria.true } };
|
|
442
411
|
},
|
|
443
|
-
onFailure: (error) => ({
|
|
444
|
-
...base,
|
|
445
|
-
verdict: "inconclusive",
|
|
446
|
-
reason: error.message,
|
|
447
|
-
unjudged: error._tag === "Declined" ? "declined" : "request too large"
|
|
448
|
-
})
|
|
412
|
+
onFailure: (error) => ({ ...base, verdict: "inconclusive", reason: error.message })
|
|
449
413
|
});
|
|
450
414
|
var itemLocation = (item) => ({
|
|
451
415
|
item: item.id,
|
|
@@ -454,23 +418,13 @@ var itemLocation = (item) => ({
|
|
|
454
418
|
});
|
|
455
419
|
var byLocation = (a, b) => (a.path ?? "").localeCompare(b.path ?? "") || (a.lines?.[0] ?? 0) - (b.lines?.[0] ?? 0) || a.id.localeCompare(b.id);
|
|
456
420
|
var select = Effect4.fn("select")(function* (policy, cwd, runs) {
|
|
457
|
-
|
|
458
|
-
const pathless = items.find((item) => item.path === undefined);
|
|
459
|
-
if (policy.limit?.per === "file" && pathless !== undefined) {
|
|
460
|
-
return yield* new SelectorError({
|
|
461
|
-
message: `${policy.name}: item ${pathless.id} has no path, and limit per file counts items by path; give every item a path or use per: run`
|
|
462
|
-
});
|
|
463
|
-
}
|
|
464
|
-
return items;
|
|
421
|
+
return (yield* runSelector(policy, cwd, runs)).toSorted(byLocation);
|
|
465
422
|
});
|
|
466
423
|
var judge = Effect4.fn("judge")(function* (policy, items) {
|
|
467
424
|
const model = yield* DecisionModel;
|
|
468
425
|
return yield* model.ask(policy.name, policy.body, items.map((item) => itemQuestion(policy, item)));
|
|
469
426
|
});
|
|
470
|
-
var decide = (policy, items, answers, threshold) => {
|
|
471
|
-
const found = Arr.zipWith(items, answers, (item, violates) => toFinding(violates, { policy: policy.name, ...itemLocation(item) }, policy, threshold));
|
|
472
|
-
return policy.limit === undefined ? found : aggregate(policy.name, policy.limit, found);
|
|
473
|
-
};
|
|
427
|
+
var decide = (policy, items, answers, threshold) => Arr.zipWith(items, answers, (item, violates) => toFinding(violates, { policy: policy.name, ...itemLocation(item) }, policy, threshold));
|
|
474
428
|
var policyConcurrency = 8;
|
|
475
429
|
var lint = Effect4.fn("lint")(function* (policies, threshold) {
|
|
476
430
|
const runs = new Map;
|
|
@@ -493,10 +447,6 @@ var Frontmatter = Schema4.Struct({
|
|
|
493
447
|
criteria: Schema4.optional(Schema4.Struct({
|
|
494
448
|
true: Schema4.optional(Schema4.NonEmptyString),
|
|
495
449
|
false: Schema4.optional(Schema4.NonEmptyString)
|
|
496
|
-
})),
|
|
497
|
-
limit: Schema4.optional(Schema4.Struct({
|
|
498
|
-
max: Schema4.Int.check(Schema4.isGreaterThanOrEqualTo(0)),
|
|
499
|
-
per: Schema4.optional(Schema4.Literals(["file", "run"]))
|
|
500
450
|
}))
|
|
501
451
|
});
|
|
502
452
|
var frontmatterPattern = /^---\r?\n(.*?)\r?\n---(?:\r?\n|$)/s;
|
|
@@ -513,10 +463,10 @@ var parsePolicy = Effect5.fn("parsePolicy")(function* (name, source) {
|
|
|
513
463
|
const unknown = Object.keys(typeof parsed === "object" && parsed !== null ? parsed : {}).filter((key) => !(key in Frontmatter.fields));
|
|
514
464
|
if (unknown.length > 0) {
|
|
515
465
|
return yield* new PolicyError({
|
|
516
|
-
message: `${name}: unknown frontmatter ${unknown.join(", ")}; a policy has select
|
|
466
|
+
message: `${name}: unknown frontmatter ${unknown.join(", ")}; a policy has select and criteria, and its body is everything else the model reads`
|
|
517
467
|
});
|
|
518
468
|
}
|
|
519
|
-
const { select, criteria
|
|
469
|
+
const { select, criteria } = yield* Schema4.decodeUnknownEffect(Frontmatter)(parsed).pipe(Effect5.mapError((error) => new PolicyError({ message: `${name}: invalid frontmatter: ${error.message}` })));
|
|
520
470
|
if (select === undefined) {
|
|
521
471
|
return yield* new PolicyError({ message: `${name}: frontmatter needs a select command that picks what the rule judges` });
|
|
522
472
|
}
|
|
@@ -529,8 +479,7 @@ var parsePolicy = Effect5.fn("parsePolicy")(function* (name, source) {
|
|
|
529
479
|
name,
|
|
530
480
|
body,
|
|
531
481
|
select,
|
|
532
|
-
criteria: { true: criteria.true, false: criteria.false }
|
|
533
|
-
...limit === undefined ? {} : { limit: { max: limit.max, per: limit.per ?? "file" } }
|
|
482
|
+
criteria: { true: criteria.true, false: criteria.false }
|
|
534
483
|
};
|
|
535
484
|
});
|
|
536
485
|
var loadPolicies = Effect5.fn("loadPolicies")(function* (directory) {
|
|
@@ -784,7 +733,6 @@ export {
|
|
|
784
733
|
PolicyError,
|
|
785
734
|
RequestTooLarge,
|
|
786
735
|
SelectorError,
|
|
787
|
-
aggregate,
|
|
788
736
|
auroc,
|
|
789
737
|
bootstrapInterval,
|
|
790
738
|
brier,
|
package/dist/types/lib.d.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* - `select`: runs a policy's selector and returns its items, sorted as they are judged.
|
|
5
5
|
* - `judge`: asks a decision model whether each item violates the policy (needs the `DecisionModel` service).
|
|
6
|
-
* - `decide`: turns answers into findings,
|
|
6
|
+
* - `decide`: turns answers into findings, one per item; pure.
|
|
7
7
|
* - `lint`: all three for every policy, as the CLI runs them.
|
|
8
8
|
*
|
|
9
9
|
* `DecisionModel.ask` is the judge without policies: yes/no questions under a shared state, packed, cached,
|
|
@@ -28,7 +28,7 @@ export declare const ClefError: typeof DecisionModelError;
|
|
|
28
28
|
/** @deprecated Renamed `DecisionModelError`. */
|
|
29
29
|
export type ClefError = DecisionModelError;
|
|
30
30
|
export { type Item, parseItem } from "./item.ts";
|
|
31
|
-
export {
|
|
31
|
+
export { decide, defaultThreshold, type Finding, itemQuestion, judge, lint, reviewFloor, select, type Verdict, verdictOf } from "./lint.ts";
|
|
32
32
|
export { loadPolicies, parsePolicy, type Policy, PolicyError } from "./policy.ts";
|
|
33
33
|
export { killRunningSelectors, runSelector, SelectorError, type SelectorRuns, selectorTimeoutMinutes } from "./select.ts";
|
|
34
34
|
export { auroc, bootstrapInterval, brier, caught, cliThresholds, concordance, falseAlarms, itemLabel, itemsOf, type Label, LabelError, type LabelRecord, loadLabels, pairedInterval, parseLabels, type PolicyScore, type Quality, quality, type Range, reachedRanges, type Scored, scorePolicy, type ScoreVerdict, type Split, targetKey, targetScore, type Thresholds, tuneThresholds, verdict, verdictF1 } from "./score.ts";
|
package/dist/types/lint.d.ts
CHANGED
|
@@ -2,34 +2,18 @@ import { Effect, Result } from "effect";
|
|
|
2
2
|
import { DecisionModel, type Question, type Unanswered } from "./decision-model.ts";
|
|
3
3
|
import type { Item } from "./item.ts";
|
|
4
4
|
import type { Policy } from "./policy.ts";
|
|
5
|
-
import {
|
|
5
|
+
import { type SelectorRuns } from "./select.ts";
|
|
6
6
|
export type Verdict = "pass" | "review" | "violation" | "inconclusive";
|
|
7
|
-
/**
|
|
8
|
-
* One judgment: of one item a policy's selector emitted (`item`, its id), or of a policy's `limit` over its
|
|
9
|
-
* items (`violating`, the ids of the items that count).
|
|
10
|
-
*/
|
|
7
|
+
/** One judgment: of one item a policy's selector emitted (`item`, its id). */
|
|
11
8
|
export interface Finding {
|
|
12
9
|
readonly policy: string;
|
|
13
10
|
readonly verdict: Verdict;
|
|
11
|
+
readonly item: string;
|
|
14
12
|
readonly path?: string;
|
|
15
|
-
readonly item?: string;
|
|
16
13
|
readonly lines?: readonly [number, number];
|
|
17
14
|
readonly violates?: number;
|
|
18
15
|
readonly reason?: string;
|
|
19
|
-
readonly violating?: ReadonlyArray<string>;
|
|
20
|
-
/** Why an `inconclusive` item has no answer: its request was too large, or the model declined it. */
|
|
21
|
-
readonly unjudged?: "request too large" | "declined";
|
|
22
16
|
}
|
|
23
|
-
/**
|
|
24
|
-
* A `limit` policy's findings: its item findings counted per file (or once per run). A group is a
|
|
25
|
-
* `violation` when more than `max` of its items are violations, and `inconclusive` when it isn't but its
|
|
26
|
-
* items that weren't judged could put it over. Items in `review` are counted in the reason but not against
|
|
27
|
-
* the limit. The reason says where the violating items are, and why items weren't judged.
|
|
28
|
-
*/
|
|
29
|
-
export declare const aggregate: (policy: string, limit: {
|
|
30
|
-
readonly max: number;
|
|
31
|
-
readonly per: "file" | "run";
|
|
32
|
-
}, items: ReadonlyArray<Finding>) => Array<Finding>;
|
|
33
17
|
export declare const reviewFloor = 0.5;
|
|
34
18
|
export declare const defaultThreshold = 0.65;
|
|
35
19
|
/** The verdict for violation probability `violates` under the given cut-offs. */
|
|
@@ -45,7 +29,7 @@ export declare const select: (policy: Policy, cwd?: string | undefined, runs?: S
|
|
|
45
29
|
readonly text: string;
|
|
46
30
|
readonly path?: string | undefined;
|
|
47
31
|
readonly lines?: readonly [number, number] | undefined;
|
|
48
|
-
}[], SelectorError, never>;
|
|
32
|
+
}[], import("./select.ts").SelectorError, never>;
|
|
49
33
|
/**
|
|
50
34
|
* Judge stage: asks the decision model whether each item violates `policy`, under the policy body. Results
|
|
51
35
|
* follow `items` order: the probability that the item violates the rule, or why there is none
|
|
@@ -58,12 +42,12 @@ export declare const judge: (policy: Policy, items: readonly {
|
|
|
58
42
|
readonly lines?: readonly [number, number] | undefined;
|
|
59
43
|
}[]) => Effect.Effect<readonly Result.Result<number, Unanswered>[], import("./decision-model.ts").DecisionModelError, DecisionModel>;
|
|
60
44
|
/**
|
|
61
|
-
* Decide stage:
|
|
62
|
-
* `threshold
|
|
45
|
+
* Decide stage: one finding per item of `items`, given their `answers` (from `judge`, in the same order),
|
|
46
|
+
* under `threshold`.
|
|
63
47
|
*/
|
|
64
48
|
export declare const decide: (policy: Policy, items: ReadonlyArray<Item>, answers: ReadonlyArray<Result.Result<number, Unanswered>>, threshold: number) => Array<Finding>;
|
|
65
49
|
/**
|
|
66
50
|
* Runs each policy's selector from the current directory and judges the items it selects. Findings follow
|
|
67
51
|
* `policies` order.
|
|
68
52
|
*/
|
|
69
|
-
export declare const lint: (policies: readonly Policy[], threshold: number) => Effect.Effect<Finding[], import("./decision-model.ts").DecisionModelError | SelectorError, DecisionModel>;
|
|
53
|
+
export declare const lint: (policies: readonly Policy[], threshold: number) => Effect.Effect<Finding[], import("./decision-model.ts").DecisionModelError | import("./select.ts").SelectorError, DecisionModel>;
|
package/dist/types/policy.d.ts
CHANGED
|
@@ -4,8 +4,7 @@ import { Effect, FileSystem, Path, Runtime, Schema } from "effect";
|
|
|
4
4
|
* text, sent exactly as written (the rule, and any framing or examples the author wants). The `select` shell
|
|
5
5
|
* command finds what to judge itself and emits it as items; each item's text is one question. `criteria`
|
|
6
6
|
* says what a yes (the item violates the rule) and a no mean for every question: without them a model may read yes as
|
|
7
|
-
* "follows the rule" and invert every verdict.
|
|
8
|
-
* violate the rule, per file or per run, against a maximum.
|
|
7
|
+
* "follows the rule" and invert every verdict.
|
|
9
8
|
*/
|
|
10
9
|
export interface Policy {
|
|
11
10
|
readonly name: string;
|
|
@@ -15,10 +14,6 @@ export interface Policy {
|
|
|
15
14
|
readonly true: string;
|
|
16
15
|
readonly false: string;
|
|
17
16
|
};
|
|
18
|
-
readonly limit?: {
|
|
19
|
-
readonly max: number;
|
|
20
|
-
readonly per: "file" | "run";
|
|
21
|
-
};
|
|
22
17
|
}
|
|
23
18
|
declare const PolicyError_base: Schema.Class<PolicyError, Schema.TaggedStruct<"PolicyError", {
|
|
24
19
|
readonly message: Schema.String;
|
|
@@ -34,10 +29,6 @@ export declare const parsePolicy: (name: string, source: string) => Effect.Effec
|
|
|
34
29
|
true: string;
|
|
35
30
|
false: string;
|
|
36
31
|
};
|
|
37
|
-
limit?: {
|
|
38
|
-
max: number;
|
|
39
|
-
per: "file" | "run";
|
|
40
|
-
} | undefined;
|
|
41
32
|
}, PolicyError, never>;
|
|
42
33
|
/** Loads every `*.md` policy under `directory`. A missing or empty directory is an error. */
|
|
43
34
|
export declare const loadPolicies: (directory: string) => Effect.Effect<{
|
|
@@ -48,9 +39,5 @@ export declare const loadPolicies: (directory: string) => Effect.Effect<{
|
|
|
48
39
|
true: string;
|
|
49
40
|
false: string;
|
|
50
41
|
};
|
|
51
|
-
limit?: {
|
|
52
|
-
max: number;
|
|
53
|
-
per: "file" | "run";
|
|
54
|
-
} | undefined;
|
|
55
42
|
}[], PolicyError, FileSystem.FileSystem | Path.Path>;
|
|
56
43
|
export {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@andrueandersoncs/lint-anything",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Check anything against plain-English policies, judged by a decision model: Cloudflare's Clef, or any System One or OpenAI Decisions endpoint",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Andrue Anderson",
|
|
@@ -31,26 +31,25 @@ If `lint-anything` isn't found, install it with `bun add -d @andrueandersoncs/li
|
|
|
31
31
|
stdout has one line per non-passing finding, then a summary. Passing items are never listed; they only appear in the `pass` count.
|
|
32
32
|
|
|
33
33
|
```
|
|
34
|
-
violation - src/a.ts todo-limit 2 of 3 items violate the rule (at most 1 allowed): 1-1, 3-3
|
|
35
34
|
violation 0.97 src/a.ts:4-4 todo The comment has no tracking issue.
|
|
36
35
|
review 0.58 src/b.ts:10-22 no-flag-parameters The function uses a boolean parameter to choose between two jobs.
|
|
37
36
|
inconclusive - big explain-why request is 70123 bytes; Clef would truncate it
|
|
38
|
-
|
|
37
|
+
3 checks: 1 violation, 1 review, 1 inconclusive, 0 pass
|
|
39
38
|
```
|
|
40
39
|
|
|
41
40
|
Columns: verdict, violation probability (`-` when none), location, policy name, reason.
|
|
42
41
|
|
|
43
|
-
- **Location** is `path:start-end` when the item has `path` and `lines`,
|
|
42
|
+
- **Location** is `path:start-end` when the item has `path` and `lines`, and otherwise the item's `id` (for a commit, say).
|
|
44
43
|
- **Policy name** is the file's path under `.lint-anything/rules`, without `.md` (`go/handle-errors`). Open that file to see the full rule.
|
|
45
44
|
- **Reason:** the model returns no explanation. An item's reason is always the policy's `criteria.true`, the same text on every finding of that policy. It tells you what the policy counts as a violation, not why this particular item is one. To find out why, read the item at its location and the rule in the policy file.
|
|
46
|
-
- **"N checks"** counts findings: one per item
|
|
45
|
+
- **"N checks"** counts findings: one per item.
|
|
47
46
|
|
|
48
47
|
| Verdict | Meaning | Exit |
|
|
49
48
|
| --- | --- | --- |
|
|
50
49
|
| `pass` | probability ≤ 0.5 | 0 |
|
|
51
50
|
| `review` | between 0.5 and 0.65: uncertain, often a case reasonable reviewers would disagree on | 0 |
|
|
52
51
|
| `violation` | probability ≥ 0.65 | **1** |
|
|
53
|
-
| `inconclusive` | the item wasn't judged: too large to send (`request is N bytes; Clef would truncate it`, or `…, over the 64000-byte request limit` on another model), or refused by the model (`<model> declined to answer`, OpenAI Decisions only)
|
|
52
|
+
| `inconclusive` | the item wasn't judged: too large to send (`request is N bytes; Clef would truncate it`, or `…, over the 64000-byte request limit` on another model), or refused by the model (`<model> declined to answer`, OpenAI Decisions only) | 0 |
|
|
54
53
|
|
|
55
54
|
| Exit code | Meaning |
|
|
56
55
|
| --- | --- |
|
|
@@ -92,7 +91,7 @@ Each question shows one comment. Decide whether it breaks the rule below.
|
|
|
92
91
|
Rule: every `TODO` or `FIXME` comment must reference a tracking issue, written as `#123`, `ABC-123`, or a full issue URL.
|
|
93
92
|
```
|
|
94
93
|
|
|
95
|
-
Frontmatter keys are `select
|
|
94
|
+
Frontmatter keys are `select` and `criteria` (with both `true` and `false`), both required. Any other key stops the run (`unknown frontmatter threshold`), so there is no per-policy threshold, model, or name.
|
|
96
95
|
|
|
97
96
|
### Criteria: get the direction right
|
|
98
97
|
|
|
@@ -122,7 +121,7 @@ The selector is a shell command that finds what to judge and prints **JSONL**, o
|
|
|
122
121
|
| --- | --- | --- |
|
|
123
122
|
| `id` | yes | non-empty **string** (`3` is rejected); names the item in findings. Doesn't need to be unique |
|
|
124
123
|
| `text` | yes | non-empty string: **everything the model reads for this item**, including its context |
|
|
125
|
-
| `path` | no | file path, for the location
|
|
124
|
+
| `path` | no | file path, for the location |
|
|
126
125
|
| `lines` | no | `[start, end]`, integers, **1-based**, inclusive, `1 ≤ start ≤ end` |
|
|
127
126
|
|
|
128
127
|
Any other field, a non-JSON line, or a wrong type stops the whole run with `selector output line N is not a valid item: …`. Blank lines are skipped.
|
|
@@ -177,26 +176,9 @@ What decides accuracy is what the item's text says (from the evals):
|
|
|
177
176
|
- **Items must reach every possible violation.** A violation that no item contains can't be found.
|
|
178
177
|
- **Compute facts; don't make the model derive them.** For "TODOs older than 90 days need an issue", giving the TODO's age in days scored 99% accuracy. Giving its commit date and today's date scored 84%, and the TODO alone 76%. Clef doesn't do date math or counting. Do it in the selector, with `git blame` or a script.
|
|
179
178
|
- **Add only the context the rule turns on**: the enclosing function for a magic number, the call sites for a flag parameter, whether a symbol is used for a public-surface rule. Context costs input tokens.
|
|
180
|
-
- **Rules about a count go in `limit`**, not in the rule text (next section).
|
|
181
179
|
- **Line numbers in `text` reduce cache hits.** The cache is keyed on the text, so `"text": "src/a.ts:42: …"` gets asked again whenever a line above it is added or removed. Location is already in `path`/`lines`. Put the path in the text only if the rule needs it (test-file rules, say), and put line numbers in only if the rule needs them.
|
|
182
180
|
- **Selectors built for a rule beat generic ones.** If you reword a rule, check its selector still reaches what the new wording covers.
|
|
183
181
|
|
|
184
|
-
## Counting: `limit`
|
|
185
|
-
|
|
186
|
-
```yaml
|
|
187
|
-
limit: { max: 3, per: file } # per: file (default) or per: run; max is an integer ≥ 0
|
|
188
|
-
```
|
|
189
|
-
|
|
190
|
-
Every item is judged as usual. Then the policy reports one finding per file (or one for the whole run) instead of one per item:
|
|
191
|
-
|
|
192
|
-
- `violation` when more than `max` items in the group are violations. The reason lists where they are: their line ranges within the file (`1-1, 3-3`), or `path:lines` or ids with `per: run`.
|
|
193
|
-
- `inconclusive` when the group isn't over the limit but its unjudged (too large) items could put it over.
|
|
194
|
-
- otherwise `pass`. Items in `review` are mentioned in the reason but don't count toward the limit.
|
|
195
|
-
- With `per: file`, every item needs a `path`. One without a path stops the run.
|
|
196
|
-
- With `per: file`, a file with no items gets no finding. With `per: run`, there is always one finding, even with 0 items.
|
|
197
|
-
|
|
198
|
-
Write the rule about a single item ("every TODO references an issue") and put the number in `limit`. On the evals, counting with `limit` was exact on all 64 files; asking Clef "more than 3 per file?" about the whole file got 92%.
|
|
199
|
-
|
|
200
182
|
## Testing and tuning a policy
|
|
201
183
|
|
|
202
184
|
Tune against labeled code, not against findings you happen to see. A policy tuned until it fits the cases you looked at gets worse on the ones you didn't: on the evals, one rule's selector improved on the tuning half and lost 0.06 AUROC on the held-out half.
|
|
@@ -259,5 +241,5 @@ The lint-anything repository has worked examples: 21 policies in `evals/corpus/p
|
|
|
259
241
|
- [ ] The selector can't fail silently (no unguarded failure early in a pipe) and doesn't stop the run when nothing matches.
|
|
260
242
|
- [ ] `criteria.true` describes a **violation**, specifically. `criteria.false` covers compliant items **and** items the rule doesn't govern.
|
|
261
243
|
- [ ] The body holds only what the model should read: a one-line framing, then `Rule:` and a precise rule with exemptions.
|
|
262
|
-
- [ ] Facts are computed in the selector, not left for the model to derive.
|
|
244
|
+
- [ ] Facts are computed in the selector, not left for the model to derive.
|
|
263
245
|
- [ ] `.lint-anything/labels/<name>.jsonl` labels files on both splits (labels from two or three models, disagreements settled by the user), `score.js` reaches every labeled violation, and changes were adopted on test, not dev.
|