@andrueandersoncs/lint-anything 0.1.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,82 @@
1
+ import { Cause, Context, Effect, Layer, Result, Runtime, Schema } from "effect";
2
+ /** The Workers AI model lint-anything asks: `@cf/cloudflare/clef`. */
3
+ export declare const model = "clef";
4
+ /**
5
+ * One yes/no question: `instructions` is what Clef judges, and `criteria` say what a yes (`true`) and a no
6
+ * (`false`) mean. Clef answers with the probability of yes.
7
+ */
8
+ export interface Question {
9
+ readonly instructions: string;
10
+ readonly criteria: {
11
+ readonly true: string;
12
+ readonly false: string;
13
+ };
14
+ }
15
+ declare const ClefError_base: Schema.Class<ClefError, Schema.TaggedStruct<"ClefError", {
16
+ readonly message: Schema.String;
17
+ }>, Cause.YieldableError>;
18
+ export declare class ClefError extends ClefError_base {
19
+ readonly [Runtime.errorExitCode] = 2;
20
+ }
21
+ declare const RequestTooLarge_base: Schema.Class<RequestTooLarge, Schema.TaggedStruct<"RequestTooLarge", {
22
+ readonly bytes: Schema.Number;
23
+ }>, Cause.YieldableError>;
24
+ /** The request would exceed Clef's context window, which silently truncates state. */
25
+ export declare class RequestTooLarge extends RequestTooLarge_base {
26
+ }
27
+ /** USD per input token. Clef bills no output tokens. */
28
+ export declare const inputTokenPrice = 2.4e-7;
29
+ /**
30
+ * How long a cached answer stands. After it, the question is asked again, so answers follow updates to the
31
+ * model, and the expired answer is deleted, so the cache doesn't grow without bound.
32
+ */
33
+ export declare const cacheMaxAgeMs: number;
34
+ /** One Clef request: its exact body and what each question id answers. */
35
+ export interface Planned<A> {
36
+ readonly body: string;
37
+ readonly questions: ReadonlyArray<{
38
+ readonly id: string;
39
+ readonly value: A;
40
+ }>;
41
+ }
42
+ /**
43
+ * Plans the requests that ask `questions` under `state`: the state once per request and as many questions
44
+ * as fit. Planned questions answer by index into `questions`.
45
+ */
46
+ export declare const planRequests: (state: string, questions: ReadonlyArray<Question>) => {
47
+ requests: Planned<number>[];
48
+ oversized: {
49
+ readonly value: number;
50
+ readonly bytes: number;
51
+ }[];
52
+ };
53
+ /** What one source's questions cost in a run. */
54
+ export interface Usage {
55
+ /** Requests sent to Clef (retries of a request not counted again). */
56
+ readonly requests: number;
57
+ /** Answers taken from the cache instead of asked. */
58
+ readonly cached: number;
59
+ /** Input tokens Clef reported for the requests sent; Clef bills no output tokens. */
60
+ readonly inputTokens: number;
61
+ /** Requests whose response reported no token count, so `inputTokens` leaves them out. */
62
+ readonly unmetered: number;
63
+ }
64
+ export declare const noUsage: Usage;
65
+ export declare const sumUsage: (usages: Iterable<Usage>) => Usage;
66
+ declare const Clef_base: Context.ServiceClass<Clef, "lint-anything/Clef", {
67
+ /**
68
+ * Asks each of `questions` under the shared `state`; results follow `questions` order, each the probability
69
+ * of yes, or `RequestTooLarge` for a question too large to send. `source` names who asked, for `stats`.
70
+ */
71
+ ask(source: string, state: string, questions: ReadonlyArray<Question>): Effect.Effect<ReadonlyArray<Result.Result<number, RequestTooLarge>>, ClefError>;
72
+ /** What each source's questions cost so far, by source; a source that sent and cached nothing is absent. */
73
+ readonly stats: Effect.Effect<ReadonlyMap<string, Usage>>;
74
+ }>;
75
+ export declare class Clef extends Clef_base {
76
+ /**
77
+ * Clef on Workers AI; with `cacheDirectory`, answers are cached in a database there per question across
78
+ * runs, and answers older than `cacheMaxAgeMs` are deleted and asked again.
79
+ */
80
+ static readonly layer: (cacheDirectory?: string) => Layer.Layer<Clef, ClefError, never>;
81
+ }
82
+ export {};
@@ -0,0 +1,20 @@
1
+ import { Effect, Schema } from "effect";
2
+ declare const ItemFields: Schema.Struct<{
3
+ readonly id: Schema.NonEmptyString;
4
+ readonly text: Schema.NonEmptyString;
5
+ readonly path: Schema.optional<Schema.NonEmptyString>;
6
+ readonly lines: Schema.optional<Schema.Tuple<readonly [Schema.Int, Schema.Int]>>;
7
+ }>;
8
+ /**
9
+ * One thing a selector picked out to judge. `text` is everything Clef reads for it, context included; `id`
10
+ * names it in findings; `path` and `lines` (1-based, inclusive) say where findings point.
11
+ */
12
+ export type Item = Schema.Schema.Type<typeof ItemFields>;
13
+ /** Parses one JSONL line into an item; fails with a message saying what is wrong with it. */
14
+ export declare const parseItem: (line: string) => Effect.Effect<{
15
+ readonly id: string;
16
+ readonly text: string;
17
+ readonly path?: string | undefined;
18
+ readonly lines?: readonly [number, number] | undefined;
19
+ }, string, never>;
20
+ export {};
@@ -0,0 +1,22 @@
1
+ /**
2
+ * lint-anything as an Effect library (Bun only). The stages a lint run composes, usable on their own:
3
+ *
4
+ * - `select`: runs a policy's selector and returns its items, sorted as they are judged.
5
+ * - `judge`: asks Clef whether each item violates the policy (needs the `Clef` service).
6
+ * - `decide`: turns answers into findings, `limit` included; pure.
7
+ * - `lint`: all three for every policy, as the CLI runs them.
8
+ *
9
+ * `Clef.ask` is the judge without policies: yes/no questions under a shared state, packed, cached, and
10
+ * retried. `Clef.layer(cacheDirectory?)` reads `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_AUTH_TOKEN` (and
11
+ * optionally `CLOUDFLARE_API_BASE_URL`) from the environment. `loadPolicies` needs `FileSystem` and `Path`,
12
+ * for example from `BunServices.layer` in `@effect/platform-bun`.
13
+ *
14
+ * `scorePolicy` measures a policy against labeled code (`loadLabels`: `.lint-anything/labels/<policy>.jsonl`),
15
+ * with the metrics and bootstrap intervals the evals use; the skill's score script is built on it.
16
+ */
17
+ export { cacheMaxAgeMs, Clef, ClefError, inputTokenPrice, model, noUsage, planRequests, type Question, RequestTooLarge, sumUsage, type Usage } from "./clef.ts";
18
+ export { type Item, parseItem } from "./item.ts";
19
+ export { aggregate, decide, defaultThreshold, type Finding, itemQuestion, judge, lint, reviewFloor, select, type Verdict, verdictOf } from "./lint.ts";
20
+ export { loadPolicies, parsePolicy, type Policy, PolicyError } from "./policy.ts";
21
+ export { killRunningSelectors, runSelector, SelectorError, type SelectorRuns, selectorTimeoutMinutes } from "./select.ts";
22
+ export { auroc, bootstrapInterval, brier, caught, cliThresholds, concordance, falseAlarms, itemLabel, itemsOf, type Label, LabelError, type LabelRecord, loadLabels, pairedInterval, parseLabels, type PolicyScore, type Quality, quality, type Range, reachedRanges, type Scored, scorePolicy, type ScoreVerdict, type Split, targetKey, targetScore, type Thresholds, tuneThresholds, verdict, verdictF1 } from "./score.ts";
@@ -0,0 +1,66 @@
1
+ import { Effect, Result } from "effect";
2
+ import { Clef, type Question, type RequestTooLarge } from "./clef.ts";
3
+ import type { Item } from "./item.ts";
4
+ import type { Policy } from "./policy.ts";
5
+ import { SelectorError, type SelectorRuns } from "./select.ts";
6
+ export type Verdict = "pass" | "review" | "violation" | "inconclusive";
7
+ /**
8
+ * One judgment: of one item a policy's selector emitted (`item`, its id), or of a policy's `limit` over its
9
+ * items (`violating`, the ids of the items that count).
10
+ */
11
+ export interface Finding {
12
+ readonly policy: string;
13
+ readonly verdict: Verdict;
14
+ readonly path?: string;
15
+ readonly item?: string;
16
+ readonly lines?: readonly [number, number];
17
+ readonly violates?: number;
18
+ readonly reason?: string;
19
+ readonly violating?: ReadonlyArray<string>;
20
+ }
21
+ /**
22
+ * A `limit` policy's findings: its item findings counted per file (or once per run). A group is a
23
+ * `violation` when more than `max` of its items are violations, and `inconclusive` when it isn't but its
24
+ * items that weren't judged could put it over. Items in `review` are counted in the reason but not against
25
+ * the limit. The reason says where the violating items are.
26
+ */
27
+ export declare const aggregate: (policy: string, limit: {
28
+ readonly max: number;
29
+ readonly per: "file" | "run";
30
+ }, items: ReadonlyArray<Finding>) => Array<Finding>;
31
+ export declare const reviewFloor = 0.5;
32
+ export declare const defaultThreshold = 0.65;
33
+ /** The verdict for violation probability `violates` under the given cut-offs. */
34
+ export declare const verdictOf: (violates: number, floor: number, threshold: number) => Verdict;
35
+ /** The question that judges `item` against `policy`, asked under the policy body. */
36
+ export declare const itemQuestion: (policy: Policy, item: Item) => Question;
37
+ /**
38
+ * Select stage: runs `policy.select` from `cwd` (default: the current directory) and returns its items in
39
+ * the order they are judged. With `runs`, policies with the same selector run it once.
40
+ */
41
+ export declare const select: (policy: Policy, cwd?: string | undefined, runs?: SelectorRuns | undefined) => Effect.Effect<{
42
+ readonly id: string;
43
+ readonly text: string;
44
+ readonly path?: string | undefined;
45
+ readonly lines?: readonly [number, number] | undefined;
46
+ }[], SelectorError, never>;
47
+ /**
48
+ * Judge stage: asks Clef whether each item violates `policy`, under the policy body. Results follow
49
+ * `items` order: the probability that the item violates the rule, or `RequestTooLarge`.
50
+ */
51
+ export declare const judge: (policy: Policy, items: readonly {
52
+ readonly id: string;
53
+ readonly text: string;
54
+ readonly path?: string | undefined;
55
+ readonly lines?: readonly [number, number] | undefined;
56
+ }[]) => Effect.Effect<readonly Result.Result<number, RequestTooLarge>[], import("./clef.ts").ClefError, Clef>;
57
+ /**
58
+ * Decide stage: the findings for `items` given their `answers` (from `judge`, in the same order), under
59
+ * `threshold`; with `policy.limit`, the items' findings counted against it.
60
+ */
61
+ export declare const decide: (policy: Policy, items: ReadonlyArray<Item>, answers: ReadonlyArray<Result.Result<number, RequestTooLarge>>, threshold: number) => Array<Finding>;
62
+ /**
63
+ * Runs each policy's selector from the current directory and judges the items it selects. Findings follow
64
+ * `policies` order.
65
+ */
66
+ export declare const lint: (policies: readonly Policy[], threshold: number) => Effect.Effect<Finding[], import("./clef.ts").ClefError | SelectorError, Clef>;
@@ -0,0 +1,56 @@
1
+ import { Effect, FileSystem, Path, Runtime, Schema } from "effect";
2
+ /**
3
+ * A Markdown policy: everything Clef is asked. The body after the frontmatter is the request's text, sent
4
+ * exactly as written (the rule, and any framing or examples the author wants). The `select` shell command
5
+ * finds what to judge itself and emits it as items; each item's text is one question. `criteria` says what
6
+ * a yes (the item violates the rule) and a no mean for every question: without them Clef may read yes as
7
+ * "follows the rule" and invert every verdict. With `limit`, the policy reports how many of its items
8
+ * violate the rule, per file or per run, against a maximum.
9
+ */
10
+ export interface Policy {
11
+ readonly name: string;
12
+ readonly body: string;
13
+ readonly select: string;
14
+ readonly criteria: {
15
+ readonly true: string;
16
+ readonly false: string;
17
+ };
18
+ readonly limit?: {
19
+ readonly max: number;
20
+ readonly per: "file" | "run";
21
+ };
22
+ }
23
+ declare const PolicyError_base: Schema.Class<PolicyError, Schema.TaggedStruct<"PolicyError", {
24
+ readonly message: Schema.String;
25
+ }>, import("effect/Cause").YieldableError>;
26
+ export declare class PolicyError extends PolicyError_base {
27
+ readonly [Runtime.errorExitCode] = 2;
28
+ }
29
+ export declare const parsePolicy: (name: string, source: string) => Effect.Effect<{
30
+ name: string;
31
+ body: string;
32
+ select: string;
33
+ criteria: {
34
+ true: string;
35
+ false: string;
36
+ };
37
+ limit?: {
38
+ max: number;
39
+ per: "file" | "run";
40
+ } | undefined;
41
+ }, PolicyError, never>;
42
+ /** Loads every `*.md` policy under `directory`. A missing or empty directory is an error. */
43
+ export declare const loadPolicies: (directory: string) => Effect.Effect<{
44
+ name: string;
45
+ body: string;
46
+ select: string;
47
+ criteria: {
48
+ true: string;
49
+ false: string;
50
+ };
51
+ limit?: {
52
+ max: number;
53
+ per: "file" | "run";
54
+ } | undefined;
55
+ }[], PolicyError, FileSystem.FileSystem | Path.Path>;
56
+ export {};
@@ -0,0 +1,195 @@
1
+ import { Effect, FileSystem, Path, Runtime, Schema } from "effect";
2
+ import type { Item } from "./item.ts";
3
+ import type { Policy } from "./policy.ts";
4
+ /** What a label says about a file (or an item) against one policy. */
5
+ export type Label = "violates" | "complies" | "ambiguous";
6
+ /** Labels are tuned on (`dev`) or held out to decide with (`test`). */
7
+ export type Split = "dev" | "test";
8
+ /** A 1-based, inclusive line range. */
9
+ export type Range = readonly [number, number];
10
+ declare const LabelFields: Schema.Struct<{
11
+ readonly path: Schema.optional<Schema.NonEmptyString>;
12
+ readonly item: Schema.optional<Schema.NonEmptyString>;
13
+ readonly label: Schema.Literals<readonly ["violates", "complies", "ambiguous"]>;
14
+ readonly lines: Schema.optional<Schema.$Array<Schema.Tuple<readonly [Schema.Int, Schema.Int]>>>;
15
+ readonly split: Schema.Literals<readonly ["dev", "test"]>;
16
+ readonly note: Schema.optional<Schema.String>;
17
+ }>;
18
+ /**
19
+ * One labeled target for a policy: a file (`path`), or an item without a path (`item`, its id). A violating
20
+ * file gives the line ranges of its violations (`lines`); an item of that file is a violation when its lines
21
+ * overlap one of them.
22
+ */
23
+ export type LabelRecord = Schema.Schema.Type<typeof LabelFields>;
24
+ declare const LabelError_base: Schema.Class<LabelError, Schema.TaggedStruct<"LabelError", {
25
+ readonly message: Schema.String;
26
+ }>, import("effect/Cause").YieldableError>;
27
+ export declare class LabelError extends LabelError_base {
28
+ readonly [Runtime.errorExitCode] = 2;
29
+ }
30
+ /** Where a target is: its path, or `item <id>`. */
31
+ export declare const targetKey: (record: LabelRecord) => string;
32
+ /** Parses a labels file, JSONL with one label per non-empty line; `name` is for messages. */
33
+ export declare const parseLabels: (name: string, text: string) => Effect.Effect<{
34
+ readonly path?: string | undefined;
35
+ readonly item?: string | undefined;
36
+ readonly label: "ambiguous" | "complies" | "violates";
37
+ readonly lines?: readonly (readonly [number, number])[] | undefined;
38
+ readonly split: "dev" | "test";
39
+ readonly note?: string | undefined;
40
+ }[], LabelError, never>;
41
+ /** Loads `<directory>/<policy name>.jsonl`. */
42
+ export declare const loadLabels: (policyName: string, directory?: any) => Effect.Effect<{
43
+ readonly path?: string | undefined;
44
+ readonly item?: string | undefined;
45
+ readonly label: "ambiguous" | "complies" | "violates";
46
+ readonly lines?: readonly (readonly [number, number])[] | undefined;
47
+ readonly split: "dev" | "test";
48
+ readonly note?: string | undefined;
49
+ }[], LabelError, FileSystem.FileSystem | Path.Path>;
50
+ /**
51
+ * An item's label within its target: a violating file's item is `violates` when its lines overlap one of
52
+ * the violations (an item without lines covers the whole file) and `complies` otherwise; every other item
53
+ * takes its target's label.
54
+ */
55
+ export declare const itemLabel: (target: {
56
+ readonly label: Label;
57
+ readonly lines?: ReadonlyArray<Range>;
58
+ }, item: Item) => Label;
59
+ /** How many of `ranges` some item covers; items without lines cover every range. */
60
+ export declare const reachedRanges: (ranges: ReadonlyArray<Range>, items: ReadonlyArray<Item>) => number;
61
+ /**
62
+ * A target's score as a lint run reports it: its highest item score, 0 when the selector found nothing in
63
+ * it, and `null` when none of its items could be sent.
64
+ */
65
+ export declare const targetScore: (scores: ReadonlyArray<number | null>) => number | null;
66
+ /** The items each label applies to: a file's items by path, a pathless item by id. */
67
+ export declare const itemsOf: (record: LabelRecord, items: ReadonlyArray<Item>) => {
68
+ readonly id: string;
69
+ readonly text: string;
70
+ readonly path?: string | undefined;
71
+ readonly lines?: readonly [number, number] | undefined;
72
+ }[];
73
+ /**
74
+ * A labeled unit with a policy's violation probability (`null`: too large to send). `group` is the
75
+ * resampling unit for intervals: units in one group share requests and are not independent.
76
+ */
77
+ export interface Scored<A> {
78
+ readonly subject: A;
79
+ readonly label: Label;
80
+ readonly group: string;
81
+ readonly score: number | null;
82
+ }
83
+ /** Verdict cut-offs: `pass` at or below `floor`, `violation` at or above `threshold`, `review` between. */
84
+ export interface Thresholds {
85
+ readonly floor: number;
86
+ readonly threshold: number;
87
+ }
88
+ /** The cut-offs a lint run uses. */
89
+ export declare const cliThresholds: Thresholds;
90
+ export type ScoreVerdict = "pass" | "review" | "violation";
91
+ export declare const verdict: (score: number, { floor, threshold }: Thresholds) => ScoreVerdict;
92
+ /** Separation of clear violations from clean units (ambiguous units excluded). */
93
+ export declare const auroc: (cases: ReadonlyArray<Scored<unknown>>) => number;
94
+ /**
95
+ * Ordinal concordance: over every pair of units with different targets (clean < ambiguous < violates),
96
+ * the share where the higher target gets the higher score. Rewards placing ambiguous units between.
97
+ */
98
+ export declare const concordance: (cases: ReadonlyArray<Scored<unknown>>) => number;
99
+ /** Mean squared error of the probability against the 0 / 0.5 / 1 target; lower is better. */
100
+ export declare const brier: (cases: ReadonlyArray<Scored<unknown>>) => number;
101
+ /**
102
+ * Macro-averaged F1 of the three verdicts against the three labels (pass↔complies, review↔ambiguous,
103
+ * violation↔violates). Labels absent from both labels and predictions are skipped.
104
+ */
105
+ export declare const verdictF1: (cases: ReadonlyArray<Scored<unknown>>, thresholds: Thresholds) => number;
106
+ /** The cut-offs on a 0.025 grid that maximize `verdictF1`; in-sample, so pick them on dev only. */
107
+ export declare const tuneThresholds: (cases: ReadonlyArray<Scored<unknown>>) => Thresholds;
108
+ export interface Quality {
109
+ readonly auroc: number;
110
+ readonly concordance: number;
111
+ readonly brier: number;
112
+ readonly f1: number;
113
+ /** Share of clear violations given the `violation` verdict. */
114
+ readonly caught: number;
115
+ /** Share of clear violations given `violation` or `review`. */
116
+ readonly surfaced: number;
117
+ /** Share of ambiguous units given `review`. */
118
+ readonly ambiguousToReview: number;
119
+ /** Share of clean units given `violation` or `review`. */
120
+ readonly falseAlarms: number;
121
+ /** Units too large to send. */
122
+ readonly inconclusive: number;
123
+ }
124
+ export declare const quality: (cases: ReadonlyArray<Scored<unknown>>, thresholds: Thresholds) => Quality;
125
+ /** Share of clear violations given `violation` at the CLI's cut-offs. */
126
+ export declare const caught: (cases: ReadonlyArray<Scored<unknown>>) => number;
127
+ /** Share of clean units not passed at the CLI's cut-offs. */
128
+ export declare const falseAlarms: (cases: ReadonlyArray<Scored<unknown>>) => number;
129
+ /**
130
+ * 95% paired-bootstrap interval of `metric(candidate) − metric(baseline)`, resampling groups. Both runs must
131
+ * cover the same groups.
132
+ */
133
+ export declare const pairedInterval: <A>(baseline: ReadonlyArray<Scored<A>>, candidate: ReadonlyArray<Scored<A>>, metric: (cases: ReadonlyArray<Scored<A>>) => number, samples?: number) => readonly [number, number];
134
+ /** 95% bootstrap interval of `metric(cases)`, resampling groups; NaN resamples (no violation drawn) are dropped. */
135
+ export declare const bootstrapInterval: <A>(cases: ReadonlyArray<Scored<A>>, metric: (cases: ReadonlyArray<Scored<A>>) => number, samples?: number) => readonly [number, number];
136
+ /** What scoring a policy against its labels found. */
137
+ export interface PolicyScore {
138
+ /** Each labeled target with its score: the highest score of its items, as a lint run reports it. */
139
+ readonly targets: ReadonlyArray<Scored<LabelRecord>>;
140
+ /** Each item in a labeled target, labeled by overlap with its target's violations. */
141
+ readonly items: ReadonlyArray<Scored<Item>>;
142
+ /** Violation ranges some item covers, of all labeled violation ranges, per split. */
143
+ readonly reach: Readonly<Record<Split, {
144
+ readonly reached: number;
145
+ readonly total: number;
146
+ }>>;
147
+ }
148
+ /**
149
+ * Runs `policy`'s selector from `cwd` and judges the items that fall in a labeled target (only those: the
150
+ * rest of the codebase costs nothing). Needs the `Clef` service.
151
+ */
152
+ export declare const scorePolicy: (policy: Policy, labels: readonly {
153
+ readonly path?: string | undefined;
154
+ readonly item?: string | undefined;
155
+ readonly label: "ambiguous" | "complies" | "violates";
156
+ readonly lines?: readonly (readonly [number, number])[] | undefined;
157
+ readonly split: "dev" | "test";
158
+ readonly note?: string | undefined;
159
+ }[], cwd?: string | undefined) => Effect.Effect<{
160
+ targets: {
161
+ subject: {
162
+ readonly path?: string | undefined;
163
+ readonly item?: string | undefined;
164
+ readonly label: "ambiguous" | "complies" | "violates";
165
+ readonly lines?: readonly (readonly [number, number])[] | undefined;
166
+ readonly split: "dev" | "test";
167
+ readonly note?: string | undefined;
168
+ };
169
+ label: "ambiguous" | "complies" | "violates";
170
+ group: string;
171
+ score: number | null;
172
+ }[];
173
+ items: {
174
+ subject: {
175
+ readonly id: string;
176
+ readonly text: string;
177
+ readonly path?: string | undefined;
178
+ readonly lines?: readonly [number, number] | undefined;
179
+ };
180
+ label: Label;
181
+ group: string;
182
+ score: number | null;
183
+ }[];
184
+ reach: {
185
+ dev: {
186
+ reached: number;
187
+ total: number;
188
+ };
189
+ test: {
190
+ reached: number;
191
+ total: number;
192
+ };
193
+ };
194
+ }, import("./clef.ts").ClefError | import("./select.ts").SelectorError, import("./clef.ts").Clef>;
195
+ export {};
@@ -0,0 +1,36 @@
1
+ import { Effect, Runtime, Schema } from "effect";
2
+ import type { Policy } from "./policy.ts";
3
+ declare const SelectorError_base: Schema.Class<SelectorError, Schema.TaggedStruct<"SelectorError", {
4
+ readonly message: Schema.String;
5
+ }>, import("effect/Cause").YieldableError>;
6
+ export declare class SelectorError extends SelectorError_base {
7
+ readonly [Runtime.errorExitCode] = 2;
8
+ }
9
+ /** A selector still running after this long is killed, with everything it started, and fails the run. */
10
+ export declare const selectorTimeoutMinutes = 10;
11
+ interface SelectorOutput {
12
+ readonly exitCode: number;
13
+ readonly timedOut: boolean;
14
+ readonly stdout: string;
15
+ readonly stderr: string;
16
+ }
17
+ /**
18
+ * The selectors one lint run has started, so policies with the same selector run it once. Create one per
19
+ * run: what selectors read may change between runs.
20
+ */
21
+ export type SelectorRuns = Map<string, Promise<SelectorOutput>>;
22
+ /** Kills every selector process group still running. */
23
+ export declare const killRunningSelectors: () => void;
24
+ /**
25
+ * Runs `policy.select` with `sh -c` from `cwd`, with no arguments and an empty stdin: the selector finds
26
+ * what it judges itself. Its stdout must be JSONL: one item per non-empty line. A non-zero exit, an invalid
27
+ * line, or running longer than `selectorTimeoutMinutes` fails the run. With `runs`, a selector already run
28
+ * with the same command and directory reuses that run's output.
29
+ */
30
+ export declare const runSelector: (policy: Policy, cwd?: string | undefined, runs?: SelectorRuns | undefined) => Effect.Effect<{
31
+ readonly id: string;
32
+ readonly text: string;
33
+ readonly path?: string | undefined;
34
+ readonly lines?: readonly [number, number] | undefined;
35
+ }[], SelectorError, never>;
36
+ export {};
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@andrueandersoncs/lint-anything",
3
- "version": "0.1.1",
3
+ "version": "0.3.0",
4
4
  "description": "Check anything against plain-English policies, judged by Cloudflare's Clef decision model",
5
5
  "license": "MIT",
6
6
  "author": "Andrue Anderson",
@@ -8,14 +8,22 @@
8
8
  "type": "git",
9
9
  "url": "git+https://github.com/andrueandersoncs/lint-anything.git"
10
10
  },
11
- "homepage": "https://github.com/andrueandersoncs/lint-anything#readme",
11
+ "homepage": "https://lint-anything.dev",
12
12
  "bugs": "https://github.com/andrueandersoncs/lint-anything/issues",
13
13
  "type": "module",
14
14
  "bin": {
15
15
  "lint-anything": "dist/cli.js"
16
16
  },
17
+ "exports": {
18
+ ".": {
19
+ "types": "./dist/types/lib.d.ts",
20
+ "import": "./dist/lib.js"
21
+ }
22
+ },
17
23
  "files": [
18
24
  "dist/cli.js",
25
+ "dist/lib.js",
26
+ "dist/types",
19
27
  "skills"
20
28
  ],
21
29
  "engines": {
@@ -25,14 +33,22 @@
25
33
  "access": "public"
26
34
  },
27
35
  "scripts": {
28
- "build": "bun build index.ts --target=bun --minify --outfile dist/cli.js",
36
+ "build": "bun build index.ts --target=bun --minify --outfile dist/cli.js && bun build score.ts --target=bun --minify --outfile skills/lint-anything/scripts/score.js && bun build src/lib.ts --target=bun --external effect --external 'effect/*' --outfile dist/lib.js && rm -rf dist/types && tsc -p tsconfig.lib.json",
29
37
  "prepublishOnly": "bunx tsc --noEmit && bun run build"
30
38
  },
39
+ "peerDependencies": {
40
+ "effect": "^4.1.2"
41
+ },
42
+ "peerDependenciesMeta": {
43
+ "effect": {
44
+ "optional": true
45
+ }
46
+ },
31
47
  "devDependencies": {
32
48
  "@ast-grep/cli": "^0.51.0",
33
- "@effect/platform-bun": "4.0.5",
49
+ "@effect/platform-bun": "4.1.2",
34
50
  "@types/bun": "latest",
35
- "effect": "4.0.5",
51
+ "effect": "4.1.2",
36
52
  "typescript": "^7"
37
53
  },
38
54
  "trustedDependencies": [
@@ -198,17 +198,50 @@ Write the rule about a single item ("every TODO references an issue") and put th
198
198
 
199
199
  ## Testing and tuning a policy
200
200
 
201
+ Tune against labeled code, not against findings you happen to see. A policy tuned until it fits the cases you looked at gets worse on the ones you didn't: on the evals, one rule's selector improved on the tuning half and lost 0.06 AUROC on the held-out half.
202
+
201
203
  1. **Run the selector by hand** from the project root and read its items. This is the only way to see what Clef will be asked, and the only way to catch a selector that silently emits nothing:
202
204
  ```sh
203
205
  sh -c "$(bun -e 'const [, fm] = (await Bun.file(process.argv[1]).text()).match(/^---\r?\n(.*?)\r?\n---/s); console.log(Bun.YAML.parse(fm).select)' .lint-anything/rules/<name>.md)" < /dev/null | head
204
206
  ```
205
- The `< /dev/null` matters: without it, `rg` and similar tools search stdin instead of the directory. Count the items with `| wc -l`, and check that the known violations appear among them.
206
- 2. **Check direction and calibration** on items you know: at least one clear violation (it should come out `violation`) and one clean item (it should be absent from the output, counted under `pass`). If the clean one is flagged and the violating one isn't, your criteria are inverted.
207
- 3. **Iterate on selector, body, and criteria together.** Narrow the selector to the units the rule governs, add the context it turns on, and state exemptions in the rule. Re-run after each change; `--debug` shows what each run cost.
208
- 4. **Hold some examples back.** A policy tuned until it fits the cases you've looked at gets worse on the ones you haven't. Check a change on items you didn't tune on.
209
- 5. Scores for unchanged items stay fixed while they're cached. Items are sorted before they're packed, so a fresh run on the same items gives the same answers.
207
+ The `< /dev/null` matters: without it, `rg` and similar tools search stdin instead of the directory. Count the items with `| wc -l`.
208
+ 2. **Label code** in `.lint-anything/labels/<name>.jsonl` (the policy's name, subdirectories included). Commit it with the policy. See "Labels" below.
209
+ 3. **Score** the policy from the project root with the script in this skill's directory:
210
+ ```sh
211
+ bun <this skill's directory>/scripts/score.js <name>
212
+ ```
213
+ It runs the selector, judges only the items in labeled files (answers share the CLI's cache), and prints per split: labels, violations reached by some item, AUROC with a 95% interval, caught, surfaced, false alarms, and ambiguous files given `review`. Then it lists the **dev** files the policy gets wrong and the labeled violations no item contains. Test files are summarized but never listed: they are held out.
214
+ 4. **Change a copy, compare, then replace.** Edit a candidate copy outside the rules directory and score both:
215
+ ```sh
216
+ cp .lint-anything/rules/<name>.md /tmp/<name>.candidate.md # edit the copy
217
+ bun <this skill's directory>/scripts/score.js <name> /tmp/<name>.candidate.md
218
+ ```
219
+ The comparison gives candidate − current per split with paired 95% intervals. Look at dev to decide what to try; adopt the candidate only if **test** improves outside noise (the interval excludes 0) and nothing else on test gets worse outside noise. Copy it over the policy when it does.
220
+ 5. **Fix reach first.** A labeled violation in no item can't be found by any wording; change the selector until "violations reached" is complete.
221
+ 6. Intervals are wide with few labels. With fewer than about 10 violating and 10 clean files per split, most differences are within noise: label more instead of reading meaning into decimals.
222
+
223
+ ### Labels
224
+
225
+ One JSON object per line, one per labeled file:
226
+
227
+ ```jsonl
228
+ {"path": "src/orders/export.ts", "label": "violates", "lines": [[41, 58]], "split": "dev", "note": "flag parameter `dryRun` switches behavior"}
229
+ {"path": "src/orders/quota.ts", "label": "complies", "split": "test"}
230
+ {"path": "src/legacy/sync.ts", "label": "ambiguous", "split": "dev", "note": "models split; user: depends on whether retries count"}
231
+ {"item": "9acebc18", "label": "violates", "split": "test", "note": "for items without a path (commits): their id"}
232
+ ```
233
+
234
+ - `label` is `violates`, `complies`, or `ambiguous`. A violating file gives `lines`: the 1-based inclusive range of each violation in it. An item of that file counts as a violation when its lines overlap one of the ranges, and as clean otherwise; so labels stay valid when the selector changes.
235
+ - A file's score is its highest item score, as a lint run reports it; a file with no items scores 0.
236
+ - `split` is `dev` (tune on it) or `test` (held out; decide with it). Split by file, about half each, with violations on both sides.
237
+ - Label real code the selector covers: a sample of files, plus files you know violate. Planted violations (code edited to break the rule) are easier to catch than real ones; use them only to check reach, and say so in `note`.
238
+
239
+ **How to label.** Label each file independently with **two or three different models** (for example subagents running models from different families), each given the rule and the file and asked for a label and the violating line ranges.
240
+ - Where they agree, record the label.
241
+ - Where they disagree, or any of them calls it ambiguous, **ask the user** about that file, showing the rule, the code, and each model's reasoning. Record what the user decides; if the user says reasonable reviewers could disagree, record `ambiguous`. Ambiguous files are left out of AUROC and counted as right when they land in `review`.
242
+ - One model alone labels with its own reading of the rule, and tuning against those labels only makes the policy agree with that reading.
210
243
 
211
- The lint-anything repository has worked examples: 21 policies in `evals/corpus/policies/` and the scripts their selectors call in `evals/steps/`. Its evals (`bun evals/run.ts`, see its README) score policies against a labeled corpus. `$EVAL_STEPS` exists only inside those evals; copy a script into your project before using it.
244
+ The lint-anything repository has worked examples: 21 policies in `evals/corpus/policies/` and the scripts their selectors call in `evals/steps/`. Its evals score them the same way, against a labeled corpus. `$EVAL_STEPS` exists only inside those evals; copy a script into your project before using it.
212
245
 
213
246
  ## Cache and cost
214
247
 
@@ -226,4 +259,4 @@ The lint-anything repository has worked examples: 21 policies in `evals/corpus/p
226
259
  - [ ] `criteria.true` describes a **violation**, specifically. `criteria.false` covers compliant items **and** items the rule doesn't govern.
227
260
  - [ ] The body holds only what Clef should read: a one-line framing, then `Rule:` and a precise rule with exemptions.
228
261
  - [ ] Facts are computed in the selector, not left for Clef to derive. Counts go in `limit`.
229
- - [ ] A known violation comes out `violation`, and a known-clean item passes.
262
+ - [ ] `.lint-anything/labels/<name>.jsonl` labels files on both splits (labels from two or three models, disagreements settled by the user), `score.js` reaches every labeled violation, and changes were adopted on test, not dev.