@andrueandersoncs/lint-anything 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -58
- package/dist/cli.js +61 -61
- package/dist/lib.js +6 -58
- package/dist/types/lib.d.ts +2 -2
- package/dist/types/lint.d.ts +7 -23
- package/dist/types/policy.d.ts +1 -14
- package/package.json +2 -2
- package/skills/lint-anything/SKILL.md +14 -31
- package/skills/lint-anything/scripts/score.js +40 -40
package/dist/lib.js
CHANGED
|
@@ -400,37 +400,6 @@ var runSelector = Effect3.fn("runSelector")(function* (policy, cwd, runs) {
|
|
|
400
400
|
});
|
|
401
401
|
|
|
402
402
|
// src/lint.ts
|
|
403
|
-
var itemPlace = (finding, inFile) => finding.lines === undefined ? finding.item : `${inFile ? "" : `${finding.path}:`}${finding.lines[0]}-${finding.lines[1]}`;
|
|
404
|
-
var unjudgedCauses = ["request too large", "declined"];
|
|
405
|
-
var aggregate = (policy, limit, items) => {
|
|
406
|
-
const groups = new Map;
|
|
407
|
-
if (limit.per === "run")
|
|
408
|
-
groups.set(undefined, [...items]);
|
|
409
|
-
else
|
|
410
|
-
for (const item of items)
|
|
411
|
-
groups.set(item.path, [...groups.get(item.path) ?? [], item]);
|
|
412
|
-
return [...groups].map(([path, group]) => {
|
|
413
|
-
const violating = group.filter((item) => item.verdict === "violation");
|
|
414
|
-
const review = group.filter((item) => item.verdict === "review").length;
|
|
415
|
-
const unjudged = group.filter((item) => item.verdict === "inconclusive");
|
|
416
|
-
const verdict = violating.length > limit.max ? "violation" : violating.length + unjudged.length > limit.max ? "inconclusive" : "pass";
|
|
417
|
-
const places = violating.map((item) => itemPlace(item, path !== undefined)).join(", ");
|
|
418
|
-
return {
|
|
419
|
-
policy,
|
|
420
|
-
...path === undefined ? {} : { path },
|
|
421
|
-
verdict,
|
|
422
|
-
reason: [
|
|
423
|
-
`${violating.length} of ${group.length} items violate the rule (at most ${limit.max} allowed)${places === "" ? "" : `: ${places}`}`,
|
|
424
|
-
...review > 0 ? [`${review} more in review`] : [],
|
|
425
|
-
...unjudgedCauses.flatMap((cause) => {
|
|
426
|
-
const count = unjudged.filter((item) => (item.unjudged ?? "request too large") === cause).length;
|
|
427
|
-
return count > 0 ? [`${count} not judged (${cause})`] : [];
|
|
428
|
-
})
|
|
429
|
-
].join("; "),
|
|
430
|
-
violating: violating.map((item) => item.item)
|
|
431
|
-
};
|
|
432
|
-
});
|
|
433
|
-
};
|
|
434
403
|
var reviewFloor = 0.5;
|
|
435
404
|
var defaultThreshold = 0.65;
|
|
436
405
|
var verdictOf = (violates, floor, threshold) => violates <= floor ? "pass" : violates >= threshold ? "violation" : "review";
|
|
@@ -440,12 +409,7 @@ var toFinding = (violates, base, policy, threshold) => Result2.match(violates, {
|
|
|
440
409
|
const verdict = verdictOf(violates, reviewFloor, threshold);
|
|
441
410
|
return { ...base, verdict, violates, ...verdict === "pass" ? {} : { reason: policy.criteria.true } };
|
|
442
411
|
},
|
|
443
|
-
onFailure: (error) => ({
|
|
444
|
-
...base,
|
|
445
|
-
verdict: "inconclusive",
|
|
446
|
-
reason: error.message,
|
|
447
|
-
unjudged: error._tag === "Declined" ? "declined" : "request too large"
|
|
448
|
-
})
|
|
412
|
+
onFailure: (error) => ({ ...base, verdict: "inconclusive", reason: error.message })
|
|
449
413
|
});
|
|
450
414
|
var itemLocation = (item) => ({
|
|
451
415
|
item: item.id,
|
|
@@ -454,23 +418,13 @@ var itemLocation = (item) => ({
|
|
|
454
418
|
});
|
|
455
419
|
var byLocation = (a, b) => (a.path ?? "").localeCompare(b.path ?? "") || (a.lines?.[0] ?? 0) - (b.lines?.[0] ?? 0) || a.id.localeCompare(b.id);
|
|
456
420
|
var select = Effect4.fn("select")(function* (policy, cwd, runs) {
|
|
457
|
-
|
|
458
|
-
const pathless = items.find((item) => item.path === undefined);
|
|
459
|
-
if (policy.limit?.per === "file" && pathless !== undefined) {
|
|
460
|
-
return yield* new SelectorError({
|
|
461
|
-
message: `${policy.name}: item ${pathless.id} has no path, and limit per file counts items by path; give every item a path or use per: run`
|
|
462
|
-
});
|
|
463
|
-
}
|
|
464
|
-
return items;
|
|
421
|
+
return (yield* runSelector(policy, cwd, runs)).toSorted(byLocation);
|
|
465
422
|
});
|
|
466
423
|
var judge = Effect4.fn("judge")(function* (policy, items) {
|
|
467
424
|
const model = yield* DecisionModel;
|
|
468
425
|
return yield* model.ask(policy.name, policy.body, items.map((item) => itemQuestion(policy, item)));
|
|
469
426
|
});
|
|
470
|
-
var decide = (policy, items, answers, threshold) => {
|
|
471
|
-
const found = Arr.zipWith(items, answers, (item, violates) => toFinding(violates, { policy: policy.name, ...itemLocation(item) }, policy, threshold));
|
|
472
|
-
return policy.limit === undefined ? found : aggregate(policy.name, policy.limit, found);
|
|
473
|
-
};
|
|
427
|
+
var decide = (policy, items, answers, threshold) => Arr.zipWith(items, answers, (item, violates) => toFinding(violates, { policy: policy.name, ...itemLocation(item) }, policy, threshold));
|
|
474
428
|
var policyConcurrency = 8;
|
|
475
429
|
var lint = Effect4.fn("lint")(function* (policies, threshold) {
|
|
476
430
|
const runs = new Map;
|
|
@@ -493,10 +447,6 @@ var Frontmatter = Schema4.Struct({
|
|
|
493
447
|
criteria: Schema4.optional(Schema4.Struct({
|
|
494
448
|
true: Schema4.optional(Schema4.NonEmptyString),
|
|
495
449
|
false: Schema4.optional(Schema4.NonEmptyString)
|
|
496
|
-
})),
|
|
497
|
-
limit: Schema4.optional(Schema4.Struct({
|
|
498
|
-
max: Schema4.Int.check(Schema4.isGreaterThanOrEqualTo(0)),
|
|
499
|
-
per: Schema4.optional(Schema4.Literals(["file", "run"]))
|
|
500
450
|
}))
|
|
501
451
|
});
|
|
502
452
|
var frontmatterPattern = /^---\r?\n(.*?)\r?\n---(?:\r?\n|$)/s;
|
|
@@ -513,10 +463,10 @@ var parsePolicy = Effect5.fn("parsePolicy")(function* (name, source) {
|
|
|
513
463
|
const unknown = Object.keys(typeof parsed === "object" && parsed !== null ? parsed : {}).filter((key) => !(key in Frontmatter.fields));
|
|
514
464
|
if (unknown.length > 0) {
|
|
515
465
|
return yield* new PolicyError({
|
|
516
|
-
message: `${name}: unknown frontmatter ${unknown.join(", ")}; a policy has select
|
|
466
|
+
message: `${name}: unknown frontmatter ${unknown.join(", ")}; a policy has select and criteria, and its body is everything else the model reads`
|
|
517
467
|
});
|
|
518
468
|
}
|
|
519
|
-
const { select, criteria
|
|
469
|
+
const { select, criteria } = yield* Schema4.decodeUnknownEffect(Frontmatter)(parsed).pipe(Effect5.mapError((error) => new PolicyError({ message: `${name}: invalid frontmatter: ${error.message}` })));
|
|
520
470
|
if (select === undefined) {
|
|
521
471
|
return yield* new PolicyError({ message: `${name}: frontmatter needs a select command that picks what the rule judges` });
|
|
522
472
|
}
|
|
@@ -529,8 +479,7 @@ var parsePolicy = Effect5.fn("parsePolicy")(function* (name, source) {
|
|
|
529
479
|
name,
|
|
530
480
|
body,
|
|
531
481
|
select,
|
|
532
|
-
criteria: { true: criteria.true, false: criteria.false }
|
|
533
|
-
...limit === undefined ? {} : { limit: { max: limit.max, per: limit.per ?? "file" } }
|
|
482
|
+
criteria: { true: criteria.true, false: criteria.false }
|
|
534
483
|
};
|
|
535
484
|
});
|
|
536
485
|
var loadPolicies = Effect5.fn("loadPolicies")(function* (directory) {
|
|
@@ -784,7 +733,6 @@ export {
|
|
|
784
733
|
PolicyError,
|
|
785
734
|
RequestTooLarge,
|
|
786
735
|
SelectorError,
|
|
787
|
-
aggregate,
|
|
788
736
|
auroc,
|
|
789
737
|
bootstrapInterval,
|
|
790
738
|
brier,
|
package/dist/types/lib.d.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* - `select`: runs a policy's selector and returns its items, sorted as they are judged.
|
|
5
5
|
* - `judge`: asks a decision model whether each item violates the policy (needs the `DecisionModel` service).
|
|
6
|
-
* - `decide`: turns answers into findings,
|
|
6
|
+
* - `decide`: turns answers into findings, one per item; pure.
|
|
7
7
|
* - `lint`: all three for every policy, as the CLI runs them.
|
|
8
8
|
*
|
|
9
9
|
* `DecisionModel.ask` is the judge without policies: yes/no questions under a shared state, packed, cached,
|
|
@@ -28,7 +28,7 @@ export declare const ClefError: typeof DecisionModelError;
|
|
|
28
28
|
/** @deprecated Renamed `DecisionModelError`. */
|
|
29
29
|
export type ClefError = DecisionModelError;
|
|
30
30
|
export { type Item, parseItem } from "./item.ts";
|
|
31
|
-
export {
|
|
31
|
+
export { decide, defaultThreshold, type Finding, itemQuestion, judge, lint, reviewFloor, select, type Verdict, verdictOf } from "./lint.ts";
|
|
32
32
|
export { loadPolicies, parsePolicy, type Policy, PolicyError } from "./policy.ts";
|
|
33
33
|
export { killRunningSelectors, runSelector, SelectorError, type SelectorRuns, selectorTimeoutMinutes } from "./select.ts";
|
|
34
34
|
export { auroc, bootstrapInterval, brier, caught, cliThresholds, concordance, falseAlarms, itemLabel, itemsOf, type Label, LabelError, type LabelRecord, loadLabels, pairedInterval, parseLabels, type PolicyScore, type Quality, quality, type Range, reachedRanges, type Scored, scorePolicy, type ScoreVerdict, type Split, targetKey, targetScore, type Thresholds, tuneThresholds, verdict, verdictF1 } from "./score.ts";
|
package/dist/types/lint.d.ts
CHANGED
|
@@ -2,34 +2,18 @@ import { Effect, Result } from "effect";
|
|
|
2
2
|
import { DecisionModel, type Question, type Unanswered } from "./decision-model.ts";
|
|
3
3
|
import type { Item } from "./item.ts";
|
|
4
4
|
import type { Policy } from "./policy.ts";
|
|
5
|
-
import {
|
|
5
|
+
import { type SelectorRuns } from "./select.ts";
|
|
6
6
|
export type Verdict = "pass" | "review" | "violation" | "inconclusive";
|
|
7
|
-
/**
|
|
8
|
-
* One judgment: of one item a policy's selector emitted (`item`, its id), or of a policy's `limit` over its
|
|
9
|
-
* items (`violating`, the ids of the items that count).
|
|
10
|
-
*/
|
|
7
|
+
/** One judgment: of one item a policy's selector emitted (`item`, its id). */
|
|
11
8
|
export interface Finding {
|
|
12
9
|
readonly policy: string;
|
|
13
10
|
readonly verdict: Verdict;
|
|
11
|
+
readonly item: string;
|
|
14
12
|
readonly path?: string;
|
|
15
|
-
readonly item?: string;
|
|
16
13
|
readonly lines?: readonly [number, number];
|
|
17
14
|
readonly violates?: number;
|
|
18
15
|
readonly reason?: string;
|
|
19
|
-
readonly violating?: ReadonlyArray<string>;
|
|
20
|
-
/** Why an `inconclusive` item has no answer: its request was too large, or the model declined it. */
|
|
21
|
-
readonly unjudged?: "request too large" | "declined";
|
|
22
16
|
}
|
|
23
|
-
/**
|
|
24
|
-
* A `limit` policy's findings: its item findings counted per file (or once per run). A group is a
|
|
25
|
-
* `violation` when more than `max` of its items are violations, and `inconclusive` when it isn't but its
|
|
26
|
-
* items that weren't judged could put it over. Items in `review` are counted in the reason but not against
|
|
27
|
-
* the limit. The reason says where the violating items are, and why items weren't judged.
|
|
28
|
-
*/
|
|
29
|
-
export declare const aggregate: (policy: string, limit: {
|
|
30
|
-
readonly max: number;
|
|
31
|
-
readonly per: "file" | "run";
|
|
32
|
-
}, items: ReadonlyArray<Finding>) => Array<Finding>;
|
|
33
17
|
export declare const reviewFloor = 0.5;
|
|
34
18
|
export declare const defaultThreshold = 0.65;
|
|
35
19
|
/** The verdict for violation probability `violates` under the given cut-offs. */
|
|
@@ -45,7 +29,7 @@ export declare const select: (policy: Policy, cwd?: string | undefined, runs?: S
|
|
|
45
29
|
readonly text: string;
|
|
46
30
|
readonly path?: string | undefined;
|
|
47
31
|
readonly lines?: readonly [number, number] | undefined;
|
|
48
|
-
}[], SelectorError, never>;
|
|
32
|
+
}[], import("./select.ts").SelectorError, never>;
|
|
49
33
|
/**
|
|
50
34
|
* Judge stage: asks the decision model whether each item violates `policy`, under the policy body. Results
|
|
51
35
|
* follow `items` order: the probability that the item violates the rule, or why there is none
|
|
@@ -58,12 +42,12 @@ export declare const judge: (policy: Policy, items: readonly {
|
|
|
58
42
|
readonly lines?: readonly [number, number] | undefined;
|
|
59
43
|
}[]) => Effect.Effect<readonly Result.Result<number, Unanswered>[], import("./decision-model.ts").DecisionModelError, DecisionModel>;
|
|
60
44
|
/**
|
|
61
|
-
* Decide stage:
|
|
62
|
-
* `threshold
|
|
45
|
+
* Decide stage: one finding per item of `items`, given their `answers` (from `judge`, in the same order),
|
|
46
|
+
* under `threshold`.
|
|
63
47
|
*/
|
|
64
48
|
export declare const decide: (policy: Policy, items: ReadonlyArray<Item>, answers: ReadonlyArray<Result.Result<number, Unanswered>>, threshold: number) => Array<Finding>;
|
|
65
49
|
/**
|
|
66
50
|
* Runs each policy's selector from the current directory and judges the items it selects. Findings follow
|
|
67
51
|
* `policies` order.
|
|
68
52
|
*/
|
|
69
|
-
export declare const lint: (policies: readonly Policy[], threshold: number) => Effect.Effect<Finding[], import("./decision-model.ts").DecisionModelError | SelectorError, DecisionModel>;
|
|
53
|
+
export declare const lint: (policies: readonly Policy[], threshold: number) => Effect.Effect<Finding[], import("./decision-model.ts").DecisionModelError | import("./select.ts").SelectorError, DecisionModel>;
|
package/dist/types/policy.d.ts
CHANGED
|
@@ -4,8 +4,7 @@ import { Effect, FileSystem, Path, Runtime, Schema } from "effect";
|
|
|
4
4
|
* text, sent exactly as written (the rule, and any framing or examples the author wants). The `select` shell
|
|
5
5
|
* command finds what to judge itself and emits it as items; each item's text is one question. `criteria`
|
|
6
6
|
* says what a yes (the item violates the rule) and a no mean for every question: without them a model may read yes as
|
|
7
|
-
* "follows the rule" and invert every verdict.
|
|
8
|
-
* violate the rule, per file or per run, against a maximum.
|
|
7
|
+
* "follows the rule" and invert every verdict.
|
|
9
8
|
*/
|
|
10
9
|
export interface Policy {
|
|
11
10
|
readonly name: string;
|
|
@@ -15,10 +14,6 @@ export interface Policy {
|
|
|
15
14
|
readonly true: string;
|
|
16
15
|
readonly false: string;
|
|
17
16
|
};
|
|
18
|
-
readonly limit?: {
|
|
19
|
-
readonly max: number;
|
|
20
|
-
readonly per: "file" | "run";
|
|
21
|
-
};
|
|
22
17
|
}
|
|
23
18
|
declare const PolicyError_base: Schema.Class<PolicyError, Schema.TaggedStruct<"PolicyError", {
|
|
24
19
|
readonly message: Schema.String;
|
|
@@ -34,10 +29,6 @@ export declare const parsePolicy: (name: string, source: string) => Effect.Effec
|
|
|
34
29
|
true: string;
|
|
35
30
|
false: string;
|
|
36
31
|
};
|
|
37
|
-
limit?: {
|
|
38
|
-
max: number;
|
|
39
|
-
per: "file" | "run";
|
|
40
|
-
} | undefined;
|
|
41
32
|
}, PolicyError, never>;
|
|
42
33
|
/** Loads every `*.md` policy under `directory`. A missing or empty directory is an error. */
|
|
43
34
|
export declare const loadPolicies: (directory: string) => Effect.Effect<{
|
|
@@ -48,9 +39,5 @@ export declare const loadPolicies: (directory: string) => Effect.Effect<{
|
|
|
48
39
|
true: string;
|
|
49
40
|
false: string;
|
|
50
41
|
};
|
|
51
|
-
limit?: {
|
|
52
|
-
max: number;
|
|
53
|
-
per: "file" | "run";
|
|
54
|
-
} | undefined;
|
|
55
42
|
}[], PolicyError, FileSystem.FileSystem | Path.Path>;
|
|
56
43
|
export {};
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@andrueandersoncs/lint-anything",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Check anything against plain-English policies, judged by
|
|
3
|
+
"version": "0.5.1",
|
|
4
|
+
"description": "Check anything against plain-English policies, judged by the decision model of your choice: any System One or OpenAI Decisions endpoint, or Workers AI",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Andrue Anderson",
|
|
7
7
|
"repository": {
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: lint-anything
|
|
3
|
-
description: Run lint-anything, act on its findings, and write or tune its policies (plain-English rules in `.lint-anything/rules/*.md`, each with a shell selector, judged by
|
|
3
|
+
description: Run lint-anything, act on its findings, and write or tune its policies (plain-English rules in `.lint-anything/rules/*.md`, each with a shell selector, judged by the decision model the project configures: any System One or OpenAI Decisions endpoint, or Workers AI). Use when a project has a `.lint-anything/` directory, when asked to lint code against semantic or prose rules, to fix lint-anything findings, or to add, debug, or tune a rule.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# lint-anything
|
|
7
7
|
|
|
8
|
-
lint-anything checks a codebase against rules written in English. Each rule is a **policy**: a Markdown file whose shell **selector** cuts out the items to judge (functions, comments, commits, anything), and whose body and **criteria** tell a decision model what a violation is:
|
|
8
|
+
lint-anything checks a codebase against rules written in English. Each rule is a **policy**: a Markdown file whose shell **selector** cuts out the items to judge (functions, comments, commits, anything), and whose body and **criteria** tell a decision model what a violation is: whichever model the project configures. The model returns a violation probability for each item; the CLI turns it into a verdict.
|
|
9
9
|
|
|
10
10
|
It is a guide more than a gate: semantic rules are judged probabilistically. On the project's eval corpus, well-written policies catch about 69% of real violations outright, surface about 85% (counting `review`), and raise false alarms on about 13% of clean cases. Read findings critically; don't assume every `violation` is real or every `pass` is clean.
|
|
11
11
|
|
|
@@ -22,8 +22,9 @@ If `lint-anything` isn't found, install it with `bun add -d @andrueandersoncs/li
|
|
|
22
22
|
|
|
23
23
|
- **No arguments.** Every run checks every policy; you can't pass files, a rule name, a threshold, or an output format. To check less, change the selector. The only flag is `--debug`; `--help` and `--version` also exist.
|
|
24
24
|
- **The working directory matters.** Policies are read from `./.lint-anything/rules`, and selectors run in the current directory. Run from a subdirectory and you get `error: cannot read policies in .lint-anything/rules`: there is no upward search.
|
|
25
|
-
- **
|
|
26
|
-
- **
|
|
25
|
+
- **The decision model** is the project's choice, set in the environment or in `.env` / `.env.local` in the current directory. Don't change it to make findings go away. With `LINT_ANYTHING_ENDPOINT` (a full URL), the model named by `LINT_ANYTHING_MODEL` is asked there, and `LINT_ANYTHING_API_KEY` is its bearer token if it needs one. An endpoint whose path ends in `/decisions` speaks OpenAI's Decisions API; any other speaks System One (`/v1/systemone`: Jev, LiteLLM, LLM Gateway, and most decision-model servers).
|
|
26
|
+
- **Without an endpoint**, a Cloudflare model on Workers AI is asked: `clef`, or another that `LINT_ANYTHING_MODEL` names (`clef-flash`). `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_AUTH_TOKEN` are then required on every run, even when every answer would come from the cache. `CLOUDFLARE_API_BASE_URL` is optional.
|
|
27
|
+
- **Accuracy numbers in this skill were measured on Clef.** A project on another model should score it on its labels before trusting it.
|
|
27
28
|
- **Add `.lint-anything/cache` to `.gitignore`.** The answer cache (`answers.sqlite` plus `-wal`/`-shm` files) is written there.
|
|
28
29
|
|
|
29
30
|
### Output
|
|
@@ -31,26 +32,25 @@ If `lint-anything` isn't found, install it with `bun add -d @andrueandersoncs/li
|
|
|
31
32
|
stdout has one line per non-passing finding, then a summary. Passing items are never listed; they only appear in the `pass` count.
|
|
32
33
|
|
|
33
34
|
```
|
|
34
|
-
violation - src/a.ts todo-limit 2 of 3 items violate the rule (at most 1 allowed): 1-1, 3-3
|
|
35
35
|
violation 0.97 src/a.ts:4-4 todo The comment has no tracking issue.
|
|
36
36
|
review 0.58 src/b.ts:10-22 no-flag-parameters The function uses a boolean parameter to choose between two jobs.
|
|
37
37
|
inconclusive - big explain-why request is 70123 bytes; Clef would truncate it
|
|
38
|
-
|
|
38
|
+
3 checks: 1 violation, 1 review, 1 inconclusive, 0 pass
|
|
39
39
|
```
|
|
40
40
|
|
|
41
41
|
Columns: verdict, violation probability (`-` when none), location, policy name, reason.
|
|
42
42
|
|
|
43
|
-
- **Location** is `path:start-end` when the item has `path` and `lines`,
|
|
43
|
+
- **Location** is `path:start-end` when the item has `path` and `lines`, and otherwise the item's `id` (for a commit, say).
|
|
44
44
|
- **Policy name** is the file's path under `.lint-anything/rules`, without `.md` (`go/handle-errors`). Open that file to see the full rule.
|
|
45
45
|
- **Reason:** the model returns no explanation. An item's reason is always the policy's `criteria.true`, the same text on every finding of that policy. It tells you what the policy counts as a violation, not why this particular item is one. To find out why, read the item at its location and the rule in the policy file.
|
|
46
|
-
- **"N checks"** counts findings: one per item
|
|
46
|
+
- **"N checks"** counts findings: one per item.
|
|
47
47
|
|
|
48
48
|
| Verdict | Meaning | Exit |
|
|
49
49
|
| --- | --- | --- |
|
|
50
50
|
| `pass` | probability ≤ 0.5 | 0 |
|
|
51
51
|
| `review` | between 0.5 and 0.65: uncertain, often a case reasonable reviewers would disagree on | 0 |
|
|
52
52
|
| `violation` | probability ≥ 0.65 | **1** |
|
|
53
|
-
| `inconclusive` | the item wasn't judged: too large to send (`request is N bytes; Clef would truncate it`, or `…, over the 64000-byte request limit` on another model), or refused by the model (`<model> declined to answer`, OpenAI Decisions only)
|
|
53
|
+
| `inconclusive` | the item wasn't judged: too large to send (`request is N bytes; Clef would truncate it`, or `…, over the 64000-byte request limit` on another model), or refused by the model (`<model> declined to answer`, OpenAI Decisions only) | 0 |
|
|
54
54
|
|
|
55
55
|
| Exit code | Meaning |
|
|
56
56
|
| --- | --- |
|
|
@@ -92,7 +92,7 @@ Each question shows one comment. Decide whether it breaks the rule below.
|
|
|
92
92
|
Rule: every `TODO` or `FIXME` comment must reference a tracking issue, written as `#123`, `ABC-123`, or a full issue URL.
|
|
93
93
|
```
|
|
94
94
|
|
|
95
|
-
Frontmatter keys are `select
|
|
95
|
+
Frontmatter keys are `select` and `criteria` (with both `true` and `false`), both required. Any other key stops the run (`unknown frontmatter threshold`), so there is no per-policy threshold, model, or name.
|
|
96
96
|
|
|
97
97
|
### Criteria: get the direction right
|
|
98
98
|
|
|
@@ -122,7 +122,7 @@ The selector is a shell command that finds what to judge and prints **JSONL**, o
|
|
|
122
122
|
| --- | --- | --- |
|
|
123
123
|
| `id` | yes | non-empty **string** (`3` is rejected); names the item in findings. Doesn't need to be unique |
|
|
124
124
|
| `text` | yes | non-empty string: **everything the model reads for this item**, including its context |
|
|
125
|
-
| `path` | no | file path, for the location
|
|
125
|
+
| `path` | no | file path, for the location |
|
|
126
126
|
| `lines` | no | `[start, end]`, integers, **1-based**, inclusive, `1 ≤ start ≤ end` |
|
|
127
127
|
|
|
128
128
|
Any other field, a non-JSON line, or a wrong type stops the whole run with `selector output line N is not a valid item: …`. Blank lines are skipped.
|
|
@@ -175,28 +175,11 @@ Selectors can select things other than files: `git log`, an API, a database. Use
|
|
|
175
175
|
What decides accuracy is what the item's text says (from the evals):
|
|
176
176
|
|
|
177
177
|
- **Items must reach every possible violation.** A violation that no item contains can't be found.
|
|
178
|
-
- **Compute facts; don't make the model derive them.** For "TODOs older than 90 days need an issue", giving the TODO's age in days scored 99% accuracy. Giving its commit date and today's date scored 84%, and the TODO alone 76%.
|
|
178
|
+
- **Compute facts; don't make the model derive them.** For "TODOs older than 90 days need an issue", giving the TODO's age in days scored 99% accuracy. Giving its commit date and today's date scored 84%, and the TODO alone 76%. Don't rely on the model for date math or counting (Clef got both wrong on the evals). Do it in the selector, with `git blame` or a script.
|
|
179
179
|
- **Add only the context the rule turns on**: the enclosing function for a magic number, the call sites for a flag parameter, whether a symbol is used for a public-surface rule. Context costs input tokens.
|
|
180
|
-
- **Rules about a count go in `limit`**, not in the rule text (next section).
|
|
181
180
|
- **Line numbers in `text` reduce cache hits.** The cache is keyed on the text, so `"text": "src/a.ts:42: …"` gets asked again whenever a line above it is added or removed. Location is already in `path`/`lines`. Put the path in the text only if the rule needs it (test-file rules, say), and put line numbers in only if the rule needs them.
|
|
182
181
|
- **Selectors built for a rule beat generic ones.** If you reword a rule, check its selector still reaches what the new wording covers.
|
|
183
182
|
|
|
184
|
-
## Counting: `limit`
|
|
185
|
-
|
|
186
|
-
```yaml
|
|
187
|
-
limit: { max: 3, per: file } # per: file (default) or per: run; max is an integer ≥ 0
|
|
188
|
-
```
|
|
189
|
-
|
|
190
|
-
Every item is judged as usual. Then the policy reports one finding per file (or one for the whole run) instead of one per item:
|
|
191
|
-
|
|
192
|
-
- `violation` when more than `max` items in the group are violations. The reason lists where they are: their line ranges within the file (`1-1, 3-3`), or `path:lines` or ids with `per: run`.
|
|
193
|
-
- `inconclusive` when the group isn't over the limit but its unjudged (too large) items could put it over.
|
|
194
|
-
- otherwise `pass`. Items in `review` are mentioned in the reason but don't count toward the limit.
|
|
195
|
-
- With `per: file`, every item needs a `path`. One without a path stops the run.
|
|
196
|
-
- With `per: file`, a file with no items gets no finding. With `per: run`, there is always one finding, even with 0 items.
|
|
197
|
-
|
|
198
|
-
Write the rule about a single item ("every TODO references an issue") and put the number in `limit`. On the evals, counting with `limit` was exact on all 64 files; asking Clef "more than 3 per file?" about the whole file got 92%.
|
|
199
|
-
|
|
200
183
|
## Testing and tuning a policy
|
|
201
184
|
|
|
202
185
|
Tune against labeled code, not against findings you happen to see. A policy tuned until it fits the cases you looked at gets worse on the ones you didn't: on the evals, one rule's selector improved on the tuning half and lost 0.06 AUROC on the held-out half.
|
|
@@ -249,7 +232,7 @@ The lint-anything repository has worked examples: 21 policies in `evals/corpus/p
|
|
|
249
232
|
- Each answer is cached per question, keyed by model (and endpoint), policy body, item text, and criteria. Switching models never reuses another model's answers. Re-running with nothing changed sends no requests, and an edit re-asks only the items whose text changed. Editing a policy's body or criteria re-asks all of that policy's items.
|
|
250
233
|
- Answers expire after 30 days, so everything is asked again about once a month, and verdicts can shift slightly if the model has been updated.
|
|
251
234
|
- A machine without the cache (a fresh CI runner) pays for every item on every run, unless CI keeps `.lint-anything/cache` between runs.
|
|
252
|
-
-
|
|
235
|
+
- Cost depends on the model. `--debug` prices only Clef ($0.24 per million input tokens; output is free); on another model its cost column is `-`, and the project's provider bills it. On the evals, with Clef, rule-specific selectors cost about $14–16 per 1,000 files across 17 rules. Nothing caps spending: a selector that emits every line of a large repository sends all of them, so check the item count by hand first.
|
|
253
236
|
- Deleting `.lint-anything/cache` forces a fresh judgment of everything.
|
|
254
237
|
|
|
255
238
|
## Quick checklist for a new policy
|
|
@@ -259,5 +242,5 @@ The lint-anything repository has worked examples: 21 policies in `evals/corpus/p
|
|
|
259
242
|
- [ ] The selector can't fail silently (no unguarded failure early in a pipe) and doesn't stop the run when nothing matches.
|
|
260
243
|
- [ ] `criteria.true` describes a **violation**, specifically. `criteria.false` covers compliant items **and** items the rule doesn't govern.
|
|
261
244
|
- [ ] The body holds only what the model should read: a one-line framing, then `Rule:` and a precise rule with exemptions.
|
|
262
|
-
- [ ] Facts are computed in the selector, not left for the model to derive.
|
|
245
|
+
- [ ] Facts are computed in the selector, not left for the model to derive.
|
|
263
246
|
- [ ] `.lint-anything/labels/<name>.jsonl` labels files on both splits (labels from two or three models, disagreements settled by the user), `score.js` reaches every labeled violation, and changes were adopted on test, not dev.
|