@andrueandersoncs/lint-anything 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/lib.js CHANGED
@@ -400,37 +400,6 @@ var runSelector = Effect3.fn("runSelector")(function* (policy, cwd, runs) {
400
400
  });
401
401
 
402
402
  // src/lint.ts
403
- var itemPlace = (finding, inFile) => finding.lines === undefined ? finding.item : `${inFile ? "" : `${finding.path}:`}${finding.lines[0]}-${finding.lines[1]}`;
404
- var unjudgedCauses = ["request too large", "declined"];
405
- var aggregate = (policy, limit, items) => {
406
- const groups = new Map;
407
- if (limit.per === "run")
408
- groups.set(undefined, [...items]);
409
- else
410
- for (const item of items)
411
- groups.set(item.path, [...groups.get(item.path) ?? [], item]);
412
- return [...groups].map(([path, group]) => {
413
- const violating = group.filter((item) => item.verdict === "violation");
414
- const review = group.filter((item) => item.verdict === "review").length;
415
- const unjudged = group.filter((item) => item.verdict === "inconclusive");
416
- const verdict = violating.length > limit.max ? "violation" : violating.length + unjudged.length > limit.max ? "inconclusive" : "pass";
417
- const places = violating.map((item) => itemPlace(item, path !== undefined)).join(", ");
418
- return {
419
- policy,
420
- ...path === undefined ? {} : { path },
421
- verdict,
422
- reason: [
423
- `${violating.length} of ${group.length} items violate the rule (at most ${limit.max} allowed)${places === "" ? "" : `: ${places}`}`,
424
- ...review > 0 ? [`${review} more in review`] : [],
425
- ...unjudgedCauses.flatMap((cause) => {
426
- const count = unjudged.filter((item) => (item.unjudged ?? "request too large") === cause).length;
427
- return count > 0 ? [`${count} not judged (${cause})`] : [];
428
- })
429
- ].join("; "),
430
- violating: violating.map((item) => item.item)
431
- };
432
- });
433
- };
434
403
  var reviewFloor = 0.5;
435
404
  var defaultThreshold = 0.65;
436
405
  var verdictOf = (violates, floor, threshold) => violates <= floor ? "pass" : violates >= threshold ? "violation" : "review";
@@ -440,12 +409,7 @@ var toFinding = (violates, base, policy, threshold) => Result2.match(violates, {
440
409
  const verdict = verdictOf(violates, reviewFloor, threshold);
441
410
  return { ...base, verdict, violates, ...verdict === "pass" ? {} : { reason: policy.criteria.true } };
442
411
  },
443
- onFailure: (error) => ({
444
- ...base,
445
- verdict: "inconclusive",
446
- reason: error.message,
447
- unjudged: error._tag === "Declined" ? "declined" : "request too large"
448
- })
412
+ onFailure: (error) => ({ ...base, verdict: "inconclusive", reason: error.message })
449
413
  });
450
414
  var itemLocation = (item) => ({
451
415
  item: item.id,
@@ -454,23 +418,13 @@ var itemLocation = (item) => ({
454
418
  });
455
419
  var byLocation = (a, b) => (a.path ?? "").localeCompare(b.path ?? "") || (a.lines?.[0] ?? 0) - (b.lines?.[0] ?? 0) || a.id.localeCompare(b.id);
456
420
  var select = Effect4.fn("select")(function* (policy, cwd, runs) {
457
- const items = (yield* runSelector(policy, cwd, runs)).toSorted(byLocation);
458
- const pathless = items.find((item) => item.path === undefined);
459
- if (policy.limit?.per === "file" && pathless !== undefined) {
460
- return yield* new SelectorError({
461
- message: `${policy.name}: item ${pathless.id} has no path, and limit per file counts items by path; give every item a path or use per: run`
462
- });
463
- }
464
- return items;
421
+ return (yield* runSelector(policy, cwd, runs)).toSorted(byLocation);
465
422
  });
466
423
  var judge = Effect4.fn("judge")(function* (policy, items) {
467
424
  const model = yield* DecisionModel;
468
425
  return yield* model.ask(policy.name, policy.body, items.map((item) => itemQuestion(policy, item)));
469
426
  });
470
- var decide = (policy, items, answers, threshold) => {
471
- const found = Arr.zipWith(items, answers, (item, violates) => toFinding(violates, { policy: policy.name, ...itemLocation(item) }, policy, threshold));
472
- return policy.limit === undefined ? found : aggregate(policy.name, policy.limit, found);
473
- };
427
+ var decide = (policy, items, answers, threshold) => Arr.zipWith(items, answers, (item, violates) => toFinding(violates, { policy: policy.name, ...itemLocation(item) }, policy, threshold));
474
428
  var policyConcurrency = 8;
475
429
  var lint = Effect4.fn("lint")(function* (policies, threshold) {
476
430
  const runs = new Map;
@@ -493,10 +447,6 @@ var Frontmatter = Schema4.Struct({
493
447
  criteria: Schema4.optional(Schema4.Struct({
494
448
  true: Schema4.optional(Schema4.NonEmptyString),
495
449
  false: Schema4.optional(Schema4.NonEmptyString)
496
- })),
497
- limit: Schema4.optional(Schema4.Struct({
498
- max: Schema4.Int.check(Schema4.isGreaterThanOrEqualTo(0)),
499
- per: Schema4.optional(Schema4.Literals(["file", "run"]))
500
450
  }))
501
451
  });
502
452
  var frontmatterPattern = /^---\r?\n(.*?)\r?\n---(?:\r?\n|$)/s;
@@ -513,10 +463,10 @@ var parsePolicy = Effect5.fn("parsePolicy")(function* (name, source) {
513
463
  const unknown = Object.keys(typeof parsed === "object" && parsed !== null ? parsed : {}).filter((key) => !(key in Frontmatter.fields));
514
464
  if (unknown.length > 0) {
515
465
  return yield* new PolicyError({
516
- message: `${name}: unknown frontmatter ${unknown.join(", ")}; a policy has select, criteria, and limit, and its body is everything else the model reads`
466
+ message: `${name}: unknown frontmatter ${unknown.join(", ")}; a policy has select and criteria, and its body is everything else the model reads`
517
467
  });
518
468
  }
519
- const { select, criteria, limit } = yield* Schema4.decodeUnknownEffect(Frontmatter)(parsed).pipe(Effect5.mapError((error) => new PolicyError({ message: `${name}: invalid frontmatter: ${error.message}` })));
469
+ const { select, criteria } = yield* Schema4.decodeUnknownEffect(Frontmatter)(parsed).pipe(Effect5.mapError((error) => new PolicyError({ message: `${name}: invalid frontmatter: ${error.message}` })));
520
470
  if (select === undefined) {
521
471
  return yield* new PolicyError({ message: `${name}: frontmatter needs a select command that picks what the rule judges` });
522
472
  }
@@ -529,8 +479,7 @@ var parsePolicy = Effect5.fn("parsePolicy")(function* (name, source) {
529
479
  name,
530
480
  body,
531
481
  select,
532
- criteria: { true: criteria.true, false: criteria.false },
533
- ...limit === undefined ? {} : { limit: { max: limit.max, per: limit.per ?? "file" } }
482
+ criteria: { true: criteria.true, false: criteria.false }
534
483
  };
535
484
  });
536
485
  var loadPolicies = Effect5.fn("loadPolicies")(function* (directory) {
@@ -784,7 +733,6 @@ export {
784
733
  PolicyError,
785
734
  RequestTooLarge,
786
735
  SelectorError,
787
- aggregate,
788
736
  auroc,
789
737
  bootstrapInterval,
790
738
  brier,
@@ -3,7 +3,7 @@
3
3
  *
4
4
  * - `select`: runs a policy's selector and returns its items, sorted as they are judged.
5
5
  * - `judge`: asks a decision model whether each item violates the policy (needs the `DecisionModel` service).
6
- * - `decide`: turns answers into findings, `limit` included; pure.
6
+ * - `decide`: turns answers into findings, one per item; pure.
7
7
  * - `lint`: all three for every policy, as the CLI runs them.
8
8
  *
9
9
  * `DecisionModel.ask` is the judge without policies: yes/no questions under a shared state, packed, cached,
@@ -28,7 +28,7 @@ export declare const ClefError: typeof DecisionModelError;
28
28
  /** @deprecated Renamed `DecisionModelError`. */
29
29
  export type ClefError = DecisionModelError;
30
30
  export { type Item, parseItem } from "./item.ts";
31
- export { aggregate, decide, defaultThreshold, type Finding, itemQuestion, judge, lint, reviewFloor, select, type Verdict, verdictOf } from "./lint.ts";
31
+ export { decide, defaultThreshold, type Finding, itemQuestion, judge, lint, reviewFloor, select, type Verdict, verdictOf } from "./lint.ts";
32
32
  export { loadPolicies, parsePolicy, type Policy, PolicyError } from "./policy.ts";
33
33
  export { killRunningSelectors, runSelector, SelectorError, type SelectorRuns, selectorTimeoutMinutes } from "./select.ts";
34
34
  export { auroc, bootstrapInterval, brier, caught, cliThresholds, concordance, falseAlarms, itemLabel, itemsOf, type Label, LabelError, type LabelRecord, loadLabels, pairedInterval, parseLabels, type PolicyScore, type Quality, quality, type Range, reachedRanges, type Scored, scorePolicy, type ScoreVerdict, type Split, targetKey, targetScore, type Thresholds, tuneThresholds, verdict, verdictF1 } from "./score.ts";
@@ -2,34 +2,18 @@ import { Effect, Result } from "effect";
2
2
  import { DecisionModel, type Question, type Unanswered } from "./decision-model.ts";
3
3
  import type { Item } from "./item.ts";
4
4
  import type { Policy } from "./policy.ts";
5
- import { SelectorError, type SelectorRuns } from "./select.ts";
5
+ import { type SelectorRuns } from "./select.ts";
6
6
  export type Verdict = "pass" | "review" | "violation" | "inconclusive";
7
- /**
8
- * One judgment: of one item a policy's selector emitted (`item`, its id), or of a policy's `limit` over its
9
- * items (`violating`, the ids of the items that count).
10
- */
7
+ /** One judgment: of one item a policy's selector emitted (`item`, its id). */
11
8
  export interface Finding {
12
9
  readonly policy: string;
13
10
  readonly verdict: Verdict;
11
+ readonly item: string;
14
12
  readonly path?: string;
15
- readonly item?: string;
16
13
  readonly lines?: readonly [number, number];
17
14
  readonly violates?: number;
18
15
  readonly reason?: string;
19
- readonly violating?: ReadonlyArray<string>;
20
- /** Why an `inconclusive` item has no answer: its request was too large, or the model declined it. */
21
- readonly unjudged?: "request too large" | "declined";
22
16
  }
23
- /**
24
- * A `limit` policy's findings: its item findings counted per file (or once per run). A group is a
25
- * `violation` when more than `max` of its items are violations, and `inconclusive` when it isn't but its
26
- * items that weren't judged could put it over. Items in `review` are counted in the reason but not against
27
- * the limit. The reason says where the violating items are, and why items weren't judged.
28
- */
29
- export declare const aggregate: (policy: string, limit: {
30
- readonly max: number;
31
- readonly per: "file" | "run";
32
- }, items: ReadonlyArray<Finding>) => Array<Finding>;
33
17
  export declare const reviewFloor = 0.5;
34
18
  export declare const defaultThreshold = 0.65;
35
19
  /** The verdict for violation probability `violates` under the given cut-offs. */
@@ -45,7 +29,7 @@ export declare const select: (policy: Policy, cwd?: string | undefined, runs?: S
45
29
  readonly text: string;
46
30
  readonly path?: string | undefined;
47
31
  readonly lines?: readonly [number, number] | undefined;
48
- }[], SelectorError, never>;
32
+ }[], import("./select.ts").SelectorError, never>;
49
33
  /**
50
34
  * Judge stage: asks the decision model whether each item violates `policy`, under the policy body. Results
51
35
  * follow `items` order: the probability that the item violates the rule, or why there is none
@@ -58,12 +42,12 @@ export declare const judge: (policy: Policy, items: readonly {
58
42
  readonly lines?: readonly [number, number] | undefined;
59
43
  }[]) => Effect.Effect<readonly Result.Result<number, Unanswered>[], import("./decision-model.ts").DecisionModelError, DecisionModel>;
60
44
  /**
61
- * Decide stage: the findings for `items` given their `answers` (from `judge`, in the same order), under
62
- * `threshold`; with `policy.limit`, the items' findings counted against it.
45
+ * Decide stage: one finding per item of `items`, given their `answers` (from `judge`, in the same order),
46
+ * under `threshold`.
63
47
  */
64
48
  export declare const decide: (policy: Policy, items: ReadonlyArray<Item>, answers: ReadonlyArray<Result.Result<number, Unanswered>>, threshold: number) => Array<Finding>;
65
49
  /**
66
50
  * Runs each policy's selector from the current directory and judges the items it selects. Findings follow
67
51
  * `policies` order.
68
52
  */
69
- export declare const lint: (policies: readonly Policy[], threshold: number) => Effect.Effect<Finding[], import("./decision-model.ts").DecisionModelError | SelectorError, DecisionModel>;
53
+ export declare const lint: (policies: readonly Policy[], threshold: number) => Effect.Effect<Finding[], import("./decision-model.ts").DecisionModelError | import("./select.ts").SelectorError, DecisionModel>;
@@ -4,8 +4,7 @@ import { Effect, FileSystem, Path, Runtime, Schema } from "effect";
4
4
  * text, sent exactly as written (the rule, and any framing or examples the author wants). The `select` shell
5
5
  * command finds what to judge itself and emits it as items; each item's text is one question. `criteria`
6
6
  * says what a yes (the item violates the rule) and a no mean for every question: without them a model may read yes as
7
- * "follows the rule" and invert every verdict. With `limit`, the policy reports how many of its items
8
- * violate the rule, per file or per run, against a maximum.
7
+ * "follows the rule" and invert every verdict.
9
8
  */
10
9
  export interface Policy {
11
10
  readonly name: string;
@@ -15,10 +14,6 @@ export interface Policy {
15
14
  readonly true: string;
16
15
  readonly false: string;
17
16
  };
18
- readonly limit?: {
19
- readonly max: number;
20
- readonly per: "file" | "run";
21
- };
22
17
  }
23
18
  declare const PolicyError_base: Schema.Class<PolicyError, Schema.TaggedStruct<"PolicyError", {
24
19
  readonly message: Schema.String;
@@ -34,10 +29,6 @@ export declare const parsePolicy: (name: string, source: string) => Effect.Effec
34
29
  true: string;
35
30
  false: string;
36
31
  };
37
- limit?: {
38
- max: number;
39
- per: "file" | "run";
40
- } | undefined;
41
32
  }, PolicyError, never>;
42
33
  /** Loads every `*.md` policy under `directory`. A missing or empty directory is an error. */
43
34
  export declare const loadPolicies: (directory: string) => Effect.Effect<{
@@ -48,9 +39,5 @@ export declare const loadPolicies: (directory: string) => Effect.Effect<{
48
39
  true: string;
49
40
  false: string;
50
41
  };
51
- limit?: {
52
- max: number;
53
- per: "file" | "run";
54
- } | undefined;
55
42
  }[], PolicyError, FileSystem.FileSystem | Path.Path>;
56
43
  export {};
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@andrueandersoncs/lint-anything",
3
- "version": "0.4.0",
3
+ "version": "0.5.0",
4
4
  "description": "Check anything against plain-English policies, judged by a decision model: Cloudflare's Clef, or any System One or OpenAI Decisions endpoint",
5
5
  "license": "MIT",
6
6
  "author": "Andrue Anderson",
@@ -31,26 +31,25 @@ If `lint-anything` isn't found, install it with `bun add -d @andrueandersoncs/li
31
31
  stdout has one line per non-passing finding, then a summary. Passing items are never listed; they only appear in the `pass` count.
32
32
 
33
33
  ```
34
- violation - src/a.ts todo-limit 2 of 3 items violate the rule (at most 1 allowed): 1-1, 3-3
35
34
  violation 0.97 src/a.ts:4-4 todo The comment has no tracking issue.
36
35
  review 0.58 src/b.ts:10-22 no-flag-parameters The function uses a boolean parameter to choose between two jobs.
37
36
  inconclusive - big explain-why request is 70123 bytes; Clef would truncate it
38
- 4 checks: 2 violation, 1 review, 1 inconclusive, 0 pass
37
+ 3 checks: 1 violation, 1 review, 1 inconclusive, 0 pass
39
38
  ```
40
39
 
41
40
  Columns: verdict, violation probability (`-` when none), location, policy name, reason.
42
41
 
43
- - **Location** is `path:start-end` when the item has `path` and `lines`, the `path` alone for a `limit` finding, and otherwise the item's `id` (for a commit, say).
42
+ - **Location** is `path:start-end` when the item has `path` and `lines`, and otherwise the item's `id` (for a commit, say).
44
43
  - **Policy name** is the file's path under `.lint-anything/rules`, without `.md` (`go/handle-errors`). Open that file to see the full rule.
45
44
  - **Reason:** the model returns no explanation. An item's reason is always the policy's `criteria.true`, the same text on every finding of that policy. It tells you what the policy counts as a violation, not why this particular item is one. To find out why, read the item at its location and the rule in the policy file.
46
- - **"N checks"** counts findings: one per item, but one per file (or per run) for a `limit` policy.
45
+ - **"N checks"** counts findings: one per item.
47
46
 
48
47
  | Verdict | Meaning | Exit |
49
48
  | --- | --- | --- |
50
49
  | `pass` | probability ≤ 0.5 | 0 |
51
50
  | `review` | between 0.5 and 0.65: uncertain, often a case reasonable reviewers would disagree on | 0 |
52
51
  | `violation` | probability ≥ 0.65 | **1** |
53
- | `inconclusive` | the item wasn't judged: too large to send (`request is N bytes; Clef would truncate it`, or `…, over the 64000-byte request limit` on another model), or refused by the model (`<model> declined to answer`, OpenAI Decisions only); or a `limit` group whose unjudged items could put it over the limit | 0 |
52
+ | `inconclusive` | the item wasn't judged: too large to send (`request is N bytes; Clef would truncate it`, or `…, over the 64000-byte request limit` on another model), or refused by the model (`<model> declined to answer`, OpenAI Decisions only) | 0 |
54
53
 
55
54
  | Exit code | Meaning |
56
55
  | --- | --- |
@@ -92,7 +91,7 @@ Each question shows one comment. Decide whether it breaks the rule below.
92
91
  Rule: every `TODO` or `FIXME` comment must reference a tracking issue, written as `#123`, `ABC-123`, or a full issue URL.
93
92
  ```
94
93
 
95
- Frontmatter keys are `select`, `criteria` (with both `true` and `false`), and optionally `limit`. All except `limit` are required. Any other key stops the run (`unknown frontmatter threshold`), so there is no per-policy threshold, model, or name.
94
+ Frontmatter keys are `select` and `criteria` (with both `true` and `false`), both required. Any other key stops the run (`unknown frontmatter threshold`), so there is no per-policy threshold, model, or name.
96
95
 
97
96
  ### Criteria: get the direction right
98
97
 
@@ -122,7 +121,7 @@ The selector is a shell command that finds what to judge and prints **JSONL**, o
122
121
  | --- | --- | --- |
123
122
  | `id` | yes | non-empty **string** (`3` is rejected); names the item in findings. Doesn't need to be unique |
124
123
  | `text` | yes | non-empty string: **everything the model reads for this item**, including its context |
125
- | `path` | no | file path, for the location (required for `limit` with `per: file`) |
124
+ | `path` | no | file path, for the location |
126
125
  | `lines` | no | `[start, end]`, integers, **1-based**, inclusive, `1 ≤ start ≤ end` |
127
126
 
128
127
  Any other field, a non-JSON line, or a wrong type stops the whole run with `selector output line N is not a valid item: …`. Blank lines are skipped.
@@ -177,26 +176,9 @@ What decides accuracy is what the item's text says (from the evals):
177
176
  - **Items must reach every possible violation.** A violation that no item contains can't be found.
178
177
  - **Compute facts; don't make the model derive them.** For "TODOs older than 90 days need an issue", giving the TODO's age in days scored 99% accuracy. Giving its commit date and today's date scored 84%, and the TODO alone 76%. Clef doesn't do date math or counting. Do it in the selector, with `git blame` or a script.
179
178
  - **Add only the context the rule turns on**: the enclosing function for a magic number, the call sites for a flag parameter, whether a symbol is used for a public-surface rule. Context costs input tokens.
180
- - **Rules about a count go in `limit`**, not in the rule text (next section).
181
179
  - **Line numbers in `text` reduce cache hits.** The cache is keyed on the text, so `"text": "src/a.ts:42: …"` gets asked again whenever a line above it is added or removed. Location is already in `path`/`lines`. Put the path in the text only if the rule needs it (test-file rules, say), and put line numbers in only if the rule needs them.
182
180
  - **Selectors built for a rule beat generic ones.** If you reword a rule, check its selector still reaches what the new wording covers.
183
181
 
184
- ## Counting: `limit`
185
-
186
- ```yaml
187
- limit: { max: 3, per: file } # per: file (default) or per: run; max is an integer ≥ 0
188
- ```
189
-
190
- Every item is judged as usual. Then the policy reports one finding per file (or one for the whole run) instead of one per item:
191
-
192
- - `violation` when more than `max` items in the group are violations. The reason lists where they are: their line ranges within the file (`1-1, 3-3`), or `path:lines` or ids with `per: run`.
193
- - `inconclusive` when the group isn't over the limit but its unjudged (too large) items could put it over.
194
- - otherwise `pass`. Items in `review` are mentioned in the reason but don't count toward the limit.
195
- - With `per: file`, every item needs a `path`. One without a path stops the run.
196
- - With `per: file`, a file with no items gets no finding. With `per: run`, there is always one finding, even with 0 items.
197
-
198
- Write the rule about a single item ("every TODO references an issue") and put the number in `limit`. On the evals, counting with `limit` was exact on all 64 files; asking Clef "more than 3 per file?" about the whole file got 92%.
199
-
200
182
  ## Testing and tuning a policy
201
183
 
202
184
  Tune against labeled code, not against findings you happen to see. A policy tuned until it fits the cases you looked at gets worse on the ones you didn't: on the evals, one rule's selector improved on the tuning half and lost 0.06 AUROC on the held-out half.
@@ -259,5 +241,5 @@ The lint-anything repository has worked examples: 21 policies in `evals/corpus/p
259
241
  - [ ] The selector can't fail silently (no unguarded failure early in a pipe) and doesn't stop the run when nothing matches.
260
242
  - [ ] `criteria.true` describes a **violation**, specifically. `criteria.false` covers compliant items **and** items the rule doesn't govern.
261
243
  - [ ] The body holds only what the model should read: a one-line framing, then `Rule:` and a precise rule with exemptions.
262
- - [ ] Facts are computed in the selector, not left for the model to derive. Counts go in `limit`.
244
+ - [ ] Facts are computed in the selector, not left for the model to derive.
263
245
  - [ ] `.lint-anything/labels/<name>.jsonl` labels files on both splits (labels from two or three models, disagreements settled by the user), `score.js` reaches every labeled violation, and changes were adopted on test, not dev.