@andrueandersoncs/lint-anything 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/lib.js ADDED
@@ -0,0 +1,699 @@
1
+ // @bun
2
+ // src/clef.ts
3
+ import { Database } from "bun:sqlite";
4
+ import { mkdirSync } from "fs";
5
+ import { join } from "path";
6
+ import { Cause, Config, Context, Effect, flow, Layer, Ref, Result, Runtime, Schedule, Schema, Semaphore } from "effect";
7
+ import { FetchHttpClient, HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http";
8
+ var model = "clef";
9
+
10
+ class ClefError extends Schema.TaggedError()("ClefError", {
11
+ message: Schema.String
12
+ }) {
13
+ [Runtime.errorExitCode] = 2;
14
+ }
15
+
16
+ class RequestTooLarge extends Schema.TaggedError()("RequestTooLarge", {
17
+ bytes: Schema.Number
18
+ }) {
19
+ }
20
+ var maxRequestBytes = 64000;
21
+ var maxQuestions = 64;
22
+ var inputTokenPrice = 0.00000024;
23
+ var maxConcurrentRequests = 8;
24
+ var attemptTimeout = "60 seconds";
25
+ var attemptRetries = 3;
26
+ var cacheMaxAgeMs = 30 * 24 * 60 * 60 * 1000;
27
+ var cacheFile = "answers.sqlite";
28
+ var cacheBusyTimeoutMs = 1e4;
29
+ var Noul = Schema.Struct({
30
+ noul: Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 }))
31
+ });
32
+ var ClefResponse = Schema.Struct({
33
+ result: Schema.Struct({
34
+ answers: Schema.Record(Schema.String, Noul),
35
+ usage: Schema.optional(Schema.Struct({ input_tokens: Schema.Number }))
36
+ })
37
+ });
38
+ var serialize = (id, question) => {
39
+ const text = `${JSON.stringify(id)}:${JSON.stringify(question)}`;
40
+ return { id, text, bytes: Buffer.byteLength(text) };
41
+ };
42
+ var pack = (state, values, entry) => {
43
+ const head = `${JSON.stringify({ model, state }).slice(0, -1)},"questions":{`;
44
+ const tail = "}}";
45
+ const fixedBytes = Buffer.byteLength(head) + tail.length;
46
+ const limit = maxRequestBytes;
47
+ const requests = [];
48
+ const oversized = [];
49
+ let entries = [];
50
+ let questions = [];
51
+ let taken = new Set;
52
+ let bytes = fixedBytes;
53
+ const flush = () => {
54
+ if (questions.length > 0)
55
+ requests.push({ body: `${head}${entries.join(",")}${tail}`, questions });
56
+ entries = [];
57
+ questions = [];
58
+ taken = new Set;
59
+ bytes = fixedBytes;
60
+ };
61
+ for (const value of values) {
62
+ const alone = entry(value, new Set);
63
+ if (fixedBytes + alone.bytes > limit) {
64
+ oversized.push({ value, bytes: fixedBytes + alone.bytes });
65
+ continue;
66
+ }
67
+ let next = entry(value, taken);
68
+ if (questions.length === maxQuestions || bytes + ",".length + next.bytes > limit) {
69
+ flush();
70
+ next = alone;
71
+ }
72
+ bytes += (questions.length === 0 ? 0 : ",".length) + next.bytes;
73
+ entries.push(next.text);
74
+ questions.push({ id: next.id, value });
75
+ taken.add(next.id);
76
+ }
77
+ flush();
78
+ return { requests, oversized };
79
+ };
80
+ var noul = (question) => ({ type: "noul", instructions: question.instructions, criteria: question.criteria });
81
+ var planRequests = (state, questions) => pack(state, questions.map((_, index) => index), (index, taken) => serialize(`i${taken.size}`, noul(questions[index])));
82
+ var openAnswerCache = (directory, now) => {
83
+ mkdirSync(directory, { recursive: true });
84
+ const db = new Database(join(directory, cacheFile));
85
+ db.run(`PRAGMA busy_timeout = ${cacheBusyTimeoutMs}`);
86
+ db.run("PRAGMA journal_mode = WAL");
87
+ db.run(`CREATE TABLE IF NOT EXISTS answers (
88
+ key BLOB PRIMARY KEY,
89
+ noul REAL NOT NULL CHECK (noul BETWEEN 0 AND 1),
90
+ answered INTEGER NOT NULL
91
+ ) STRICT, WITHOUT ROWID`);
92
+ db.run("CREATE INDEX IF NOT EXISTS answers_answered ON answers (answered)");
93
+ db.run("DELETE FROM answers WHERE answered <= ?", [now - cacheMaxAgeMs]);
94
+ const select = db.query("SELECT noul FROM answers WHERE key = ?");
95
+ const upsert = db.query("INSERT OR REPLACE INTO answers (key, noul, answered) VALUES (?, ?, ?)");
96
+ const key = (state, question) => new Bun.CryptoHasher("sha256").update(`${model}\x00${state}\x00${JSON.stringify(question)}`).digest();
97
+ const insert = db.transaction((state, answers) => {
98
+ const answered = Date.now();
99
+ for (const { question, noul } of answers)
100
+ upsert.run(key(state, question), noul, answered);
101
+ });
102
+ const failed = (error) => new ClefError({ message: `answer cache ${join(directory, cacheFile)}: ${error instanceof Error ? error.message : String(error)}` });
103
+ return {
104
+ get: (state, questions) => Effect.try({ try: () => questions.map((question) => select.get(key(state, question))?.noul), catch: failed }),
105
+ set: (state, answers) => Effect.try({ try: () => insert(state, answers), catch: failed }),
106
+ close: () => db.close()
107
+ };
108
+ };
109
+ var noUsage = { requests: 0, cached: 0, inputTokens: 0, unmetered: 0 };
110
+ var sumUsage = (usages) => {
111
+ let sum = noUsage;
112
+ for (const usage of usages) {
113
+ sum = {
114
+ requests: sum.requests + usage.requests,
115
+ cached: sum.cached + usage.cached,
116
+ inputTokens: sum.inputTokens + usage.inputTokens,
117
+ unmetered: sum.unmetered + usage.unmetered
118
+ };
119
+ }
120
+ return sum;
121
+ };
122
+ var failureMessage = (error) => {
123
+ if (Cause.isTimeoutError(error)) {
124
+ return Effect.succeed(`Clef did not respond within ${attemptTimeout}, after ${attemptRetries} retries`);
125
+ }
126
+ if (HttpClientError.isHttpClientError(error)) {
127
+ const { reason } = error;
128
+ if (reason._tag !== "StatusCodeError") {
129
+ const cause = "cause" in reason && reason.cause instanceof Error ? `: ${reason.cause.message}` : "";
130
+ return Effect.succeed(`Clef request failed: ${error.message}${cause}`);
131
+ }
132
+ return reason.response.text.pipe(Effect.timeout("5 seconds"), Effect.map((text) => text.trim().slice(0, 500)), Effect.orElseSucceed(() => ""), Effect.map((body) => `Clef responded with HTTP ${reason.response.status}${body === "" ? "" : `: ${body}`}`));
133
+ }
134
+ return Effect.succeed(`Clef sent an unexpected response: ${error instanceof Error ? error.message : String(error)}`);
135
+ };
136
+
137
+ class Clef extends Context.Service()("lint-anything/Clef") {
138
+ static layer = (cacheDirectory) => Layer.effect(Clef, Effect.gen(function* () {
139
+ const required = (name, config) => config.pipe(Effect.mapError(() => new ClefError({ message: `${name} is not set; set it in the environment or in .env` })));
140
+ const accountId = yield* required("CLOUDFLARE_ACCOUNT_ID", Config.String("CLOUDFLARE_ACCOUNT_ID"));
141
+ const token = yield* required("CLOUDFLARE_AUTH_TOKEN", Config.Redacted("CLOUDFLARE_AUTH_TOKEN"));
142
+ const baseUrl = yield* Config.String("CLOUDFLARE_API_BASE_URL").pipe(Config.withDefault("https://api.cloudflare.com/client/v4"), Effect.mapError((error) => new ClefError({ message: `CLOUDFLARE_API_BASE_URL: ${error.message}` })));
143
+ const client = (yield* HttpClient.HttpClient).pipe(HttpClient.mapRequest(flow(HttpClientRequest.prependUrl(`${baseUrl}/accounts/${accountId}/ai/run/@cf/cloudflare`), HttpClientRequest.bearerToken(token), HttpClientRequest.acceptJson)), HttpClient.filterStatusOk, HttpClient.transformResponse(flow(Effect.tap((response) => response.text), Effect.timeout(attemptTimeout))), HttpClient.retryTransient({ schedule: Schedule.exponential(500), times: attemptRetries }));
144
+ const cache = cacheDirectory === undefined ? undefined : yield* Effect.acquireRelease(Effect.try({
145
+ try: () => openAnswerCache(cacheDirectory, Date.now()),
146
+ catch: (error) => new ClefError({ message: `cannot open the answer cache in ${cacheDirectory}: ${error instanceof Error ? error.message : String(error)}` })
147
+ }), (cache2) => Effect.sync(cache2.close));
148
+ const permits = yield* Semaphore.make(maxConcurrentRequests);
149
+ const usage = yield* Ref.make(new Map);
150
+ const record = (source, added) => Ref.update(usage, (bySource) => new Map(bySource).set(source, sumUsage([bySource.get(source) ?? noUsage, { ...noUsage, ...added }])));
151
+ const send = Effect.fn("Clef.send")(function* (source, request) {
152
+ const { result } = yield* record(source, { requests: 1 }).pipe(Effect.andThen(HttpClientRequest.post(`/${model}`).pipe(HttpClientRequest.bodyText(request.body, "application/json"), client.execute)), Effect.flatMap(HttpClientResponse.schemaBodyJson(ClefResponse)), permits.withPermit, Effect.catch((error) => Effect.flatMap(failureMessage(error), (message) => Effect.fail(new ClefError({ message })))));
153
+ yield* record(source, result.usage === undefined ? { unmetered: 1 } : { inputTokens: result.usage.input_tokens });
154
+ const answers = [];
155
+ for (const { id, value } of request.questions) {
156
+ const answer = result.answers[id];
157
+ if (answer === undefined) {
158
+ return yield* new ClefError({ message: `Clef returned no answer for question ${id}` });
159
+ }
160
+ answers.push({ value, noul: answer.noul });
161
+ }
162
+ return answers;
163
+ });
164
+ const ask = Effect.fn("Clef.ask")(function* (source, state, questions) {
165
+ const results = [];
166
+ const misses = [];
167
+ const cached = cache === undefined ? [] : yield* cache.get(state, questions.map(noul));
168
+ questions.forEach((_, index) => {
169
+ const answer = cached[index];
170
+ if (answer === undefined)
171
+ misses.push(index);
172
+ else
173
+ results[index] = Result.succeed(answer);
174
+ });
175
+ if (misses.length < questions.length)
176
+ yield* record(source, { cached: questions.length - misses.length });
177
+ const { requests, oversized } = planRequests(state, misses.map((index) => questions[index]));
178
+ for (const { value, bytes } of oversized)
179
+ results[misses[value]] = Result.fail(new RequestTooLarge({ bytes }));
180
+ yield* Effect.forEach(requests, (request) => send(source, request).pipe(Effect.tap((received) => {
181
+ for (const { value, noul: answer } of received)
182
+ results[misses[value]] = Result.succeed(answer);
183
+ return cache === undefined ? Effect.void : cache.set(state, received.map(({ value, noul: answer }) => ({ question: noul(questions[misses[value]]), noul: answer })));
184
+ })), { concurrency: "unbounded", discard: true });
185
+ return results;
186
+ });
187
+ return Clef.of({ ask, stats: Ref.get(usage) });
188
+ })).pipe(Layer.provide(FetchHttpClient.layer));
189
+ }
190
+ // src/item.ts
191
+ import { Effect as Effect2, Schema as Schema2, SchemaIssue } from "effect";
192
+ var ItemFields = Schema2.Struct({
193
+ id: Schema2.NonEmptyString,
194
+ text: Schema2.NonEmptyString,
195
+ path: Schema2.optional(Schema2.NonEmptyString),
196
+ lines: Schema2.optional(Schema2.Tuple([Schema2.Int, Schema2.Int]).check(Schema2.makeFilter(([start, end]) => 1 <= start && start <= end || "lines must be [start, end] with 1 <= start <= end")))
197
+ });
198
+ var formatIssues = SchemaIssue.makeFormatterStandardSchemaV1();
199
+ var parseItem = Effect2.fn("parseItem")(function* (line) {
200
+ const value = yield* Effect2.try({ try: () => JSON.parse(line), catch: () => "not valid JSON" });
201
+ if (typeof value !== "object" || value === null || Array.isArray(value))
202
+ return yield* Effect2.fail("not an object");
203
+ const unknown = Object.keys(value).filter((key) => !(key in ItemFields.fields));
204
+ if (unknown.length > 0) {
205
+ return yield* Effect2.fail(`unknown field ${unknown.join(", ")}; put what Clef should read in text`);
206
+ }
207
+ return yield* Schema2.decodeUnknownEffect(ItemFields)(value).pipe(Effect2.mapError((error) => formatIssues(error.issue).issues.map(({ path, message }) => `${path?.length ? `${path.join(".")}: ` : ""}${message}`).join("; ")));
208
+ });
209
+ // src/lint.ts
210
+ import { Array as Arr, Effect as Effect4, Result as Result2 } from "effect";
211
+
212
+ // src/select.ts
213
+ import { Effect as Effect3, Runtime as Runtime2, Schema as Schema3 } from "effect";
214
+ class SelectorError extends Schema3.TaggedError()("SelectorError", {
215
+ message: Schema3.String
216
+ }) {
217
+ [Runtime2.errorExitCode] = 2;
218
+ }
219
+ var selectorTimeoutMinutes = 10;
220
+ var killGroup = (pid) => {
221
+ try {
222
+ process.kill(-pid, "SIGKILL");
223
+ } catch (error) {
224
+ const { code } = error;
225
+ if (code !== "ESRCH" && code !== "EPERM")
226
+ throw error;
227
+ }
228
+ };
229
+ var running = new Set;
230
+ var killRunningSelectors = () => {
231
+ for (const pid of running)
232
+ killGroup(pid);
233
+ };
234
+ process.on("exit", killRunningSelectors);
235
+ var start = async (select, cwd) => {
236
+ const child = Bun.spawn(["sh", "-c", select], {
237
+ cwd,
238
+ env: Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith("CLOUDFLARE_"))),
239
+ stdin: "ignore",
240
+ stdout: "pipe",
241
+ stderr: "pipe",
242
+ detached: true
243
+ });
244
+ running.add(child.pid);
245
+ let timedOut = false;
246
+ const timer = setTimeout(() => {
247
+ timedOut = true;
248
+ killGroup(child.pid);
249
+ }, selectorTimeoutMinutes * 60 * 1000);
250
+ try {
251
+ const [stdout, stderr, exitCode] = await Promise.all([
252
+ new Response(child.stdout).text(),
253
+ new Response(child.stderr).text(),
254
+ child.exited
255
+ ]);
256
+ return { exitCode, timedOut, stdout, stderr };
257
+ } finally {
258
+ clearTimeout(timer);
259
+ running.delete(child.pid);
260
+ }
261
+ };
262
+ var runSelector = Effect3.fn("runSelector")(function* (policy, cwd, runs) {
263
+ const result = yield* Effect3.promise(() => {
264
+ if (runs === undefined)
265
+ return start(policy.select, cwd);
266
+ const key = JSON.stringify([policy.select, cwd ?? process.cwd()]);
267
+ let run = runs.get(key);
268
+ if (run === undefined) {
269
+ run = start(policy.select, cwd);
270
+ runs.set(key, run);
271
+ }
272
+ return run;
273
+ });
274
+ if (result.timedOut) {
275
+ return yield* new SelectorError({
276
+ message: `${policy.name}: selector did not finish within ${selectorTimeoutMinutes} minutes and was killed`
277
+ });
278
+ }
279
+ if (result.exitCode !== 0) {
280
+ return yield* new SelectorError({
281
+ message: `${policy.name}: selector exited ${result.exitCode}: ${result.stderr.trim().slice(0, 500)}`
282
+ });
283
+ }
284
+ const lines = result.stdout.split(`
285
+ `);
286
+ return yield* Effect3.forEach(lines.flatMap((line, index) => line.trim() === "" ? [] : [{ line, number: index + 1 }]), ({ line, number }) => parseItem(line).pipe(Effect3.mapError((reason) => new SelectorError({ message: `${policy.name}: selector output line ${number} is not a valid item: ${reason}` }))));
287
+ });
288
+
289
+ // src/lint.ts
290
+ var itemPlace = (finding, inFile) => finding.lines === undefined ? finding.item : `${inFile ? "" : `${finding.path}:`}${finding.lines[0]}-${finding.lines[1]}`;
291
+ var aggregate = (policy, limit, items) => {
292
+ const groups = new Map;
293
+ if (limit.per === "run")
294
+ groups.set(undefined, [...items]);
295
+ else
296
+ for (const item of items)
297
+ groups.set(item.path, [...groups.get(item.path) ?? [], item]);
298
+ return [...groups].map(([path, group]) => {
299
+ const violating = group.filter((item) => item.verdict === "violation");
300
+ const review = group.filter((item) => item.verdict === "review").length;
301
+ const unjudged = group.filter((item) => item.verdict === "inconclusive").length;
302
+ const verdict = violating.length > limit.max ? "violation" : violating.length + unjudged > limit.max ? "inconclusive" : "pass";
303
+ const places = violating.map((item) => itemPlace(item, path !== undefined)).join(", ");
304
+ return {
305
+ policy,
306
+ ...path === undefined ? {} : { path },
307
+ verdict,
308
+ reason: [
309
+ `${violating.length} of ${group.length} items violate the rule (at most ${limit.max} allowed)${places === "" ? "" : `: ${places}`}`,
310
+ ...review > 0 ? [`${review} more in review`] : [],
311
+ ...unjudged > 0 ? [`${unjudged} not judged (request too large)`] : []
312
+ ].join("; "),
313
+ violating: violating.map((item) => item.item)
314
+ };
315
+ });
316
+ };
317
+ var reviewFloor = 0.5;
318
+ var defaultThreshold = 0.65;
319
+ var verdictOf = (violates, floor, threshold) => violates <= floor ? "pass" : violates >= threshold ? "violation" : "review";
320
+ var itemQuestion = (policy, item) => ({ instructions: item.text, criteria: policy.criteria });
321
+ var toFinding = (violates, base, policy, threshold) => Result2.match(violates, {
322
+ onSuccess: (violates) => {
323
+ const verdict = verdictOf(violates, reviewFloor, threshold);
324
+ return { ...base, verdict, violates, ...verdict === "pass" ? {} : { reason: policy.criteria.true } };
325
+ },
326
+ onFailure: ({ bytes }) => ({ ...base, verdict: "inconclusive", reason: `request is ${bytes} bytes; Clef would truncate it` })
327
+ });
328
+ var itemLocation = (item) => ({
329
+ item: item.id,
330
+ ...item.path === undefined ? {} : { path: item.path },
331
+ ...item.lines === undefined ? {} : { lines: item.lines }
332
+ });
333
+ var byLocation = (a, b) => (a.path ?? "").localeCompare(b.path ?? "") || (a.lines?.[0] ?? 0) - (b.lines?.[0] ?? 0) || a.id.localeCompare(b.id);
334
+ var select = Effect4.fn("select")(function* (policy, cwd, runs) {
335
+ const items = (yield* runSelector(policy, cwd, runs)).toSorted(byLocation);
336
+ const pathless = items.find((item) => item.path === undefined);
337
+ if (policy.limit?.per === "file" && pathless !== undefined) {
338
+ return yield* new SelectorError({
339
+ message: `${policy.name}: item ${pathless.id} has no path, and limit per file counts items by path; give every item a path or use per: run`
340
+ });
341
+ }
342
+ return items;
343
+ });
344
+ var judge = Effect4.fn("judge")(function* (policy, items) {
345
+ const clef = yield* Clef;
346
+ return yield* clef.ask(policy.name, policy.body, items.map((item) => itemQuestion(policy, item)));
347
+ });
348
+ var decide = (policy, items, answers, threshold) => {
349
+ const found = Arr.zipWith(items, answers, (item, violates) => toFinding(violates, { policy: policy.name, ...itemLocation(item) }, policy, threshold));
350
+ return policy.limit === undefined ? found : aggregate(policy.name, policy.limit, found);
351
+ };
352
+ var policyConcurrency = 8;
353
+ var lint = Effect4.fn("lint")(function* (policies, threshold) {
354
+ const runs = new Map;
355
+ const findings = yield* Effect4.forEach(policies, (policy) => Effect4.gen(function* () {
356
+ const items = yield* select(policy, undefined, runs);
357
+ return decide(policy, items, yield* judge(policy, items), threshold);
358
+ }), { concurrency: policyConcurrency });
359
+ return findings.flat();
360
+ });
361
+ // src/policy.ts
362
+ import { Effect as Effect5, FileSystem, Path, Runtime as Runtime3, Schema as Schema4 } from "effect";
363
+
364
+ class PolicyError extends Schema4.TaggedError()("PolicyError", {
365
+ message: Schema4.String
366
+ }) {
367
+ [Runtime3.errorExitCode] = 2;
368
+ }
369
+ var Frontmatter = Schema4.Struct({
370
+ select: Schema4.optional(Schema4.NonEmptyString),
371
+ criteria: Schema4.optional(Schema4.Struct({
372
+ true: Schema4.optional(Schema4.NonEmptyString),
373
+ false: Schema4.optional(Schema4.NonEmptyString)
374
+ })),
375
+ limit: Schema4.optional(Schema4.Struct({
376
+ max: Schema4.Int.check(Schema4.isGreaterThanOrEqualTo(0)),
377
+ per: Schema4.optional(Schema4.Literals(["file", "run"]))
378
+ }))
379
+ });
380
+ var frontmatterPattern = /^---\r?\n(.*?)\r?\n---(?:\r?\n|$)/s;
381
+ var parsePolicy = Effect5.fn("parsePolicy")(function* (name, source) {
382
+ const match = frontmatterPattern.exec(source);
383
+ if (match === null) {
384
+ return yield* new PolicyError({ message: `${name}: missing frontmatter` });
385
+ }
386
+ const body = source.slice(match[0].length);
387
+ if (body.trim() === "") {
388
+ return yield* new PolicyError({ message: `${name}: empty policy` });
389
+ }
390
+ const parsed = yield* Effect5.try(() => Bun.YAML.parse(match[1] ?? "")).pipe(Effect5.mapError((error) => new PolicyError({ message: `${name}: invalid frontmatter: ${String(error.message ?? error)}` })));
391
+ const unknown = Object.keys(typeof parsed === "object" && parsed !== null ? parsed : {}).filter((key) => !(key in Frontmatter.fields));
392
+ if (unknown.length > 0) {
393
+ return yield* new PolicyError({
394
+ message: `${name}: unknown frontmatter ${unknown.join(", ")}; a policy has select, criteria, and limit, and its body is everything else Clef reads`
395
+ });
396
+ }
397
+ const { select, criteria, limit } = yield* Schema4.decodeUnknownEffect(Frontmatter)(parsed).pipe(Effect5.mapError((error) => new PolicyError({ message: `${name}: invalid frontmatter: ${error.message}` })));
398
+ if (select === undefined) {
399
+ return yield* new PolicyError({ message: `${name}: frontmatter needs a select command that picks what the rule judges` });
400
+ }
401
+ if (criteria?.true === undefined || criteria.false === undefined) {
402
+ return yield* new PolicyError({
403
+ message: `${name}: frontmatter needs criteria with "true" (what an item that violates the rule is) and "false" (what one that follows it is)`
404
+ });
405
+ }
406
+ return {
407
+ name,
408
+ body,
409
+ select,
410
+ criteria: { true: criteria.true, false: criteria.false },
411
+ ...limit === undefined ? {} : { limit: { max: limit.max, per: limit.per ?? "file" } }
412
+ };
413
+ });
414
+ var loadPolicies = Effect5.fn("loadPolicies")(function* (directory) {
415
+ const fs = yield* FileSystem.FileSystem;
416
+ const path = yield* Path.Path;
417
+ const unreadable = () => new PolicyError({ message: `cannot read policies in ${directory}` });
418
+ const entries = yield* fs.readDirectory(directory, { recursive: true }).pipe(Effect5.mapError(unreadable));
419
+ const files = entries.filter((entry) => entry.endsWith(".md")).sort();
420
+ if (files.length === 0) {
421
+ return yield* new PolicyError({ message: `no policies in ${directory}` });
422
+ }
423
+ return yield* Effect5.forEach(files, (file) => fs.readFileString(path.join(directory, file)).pipe(Effect5.mapError(unreadable), Effect5.flatMap((source) => parsePolicy(file.slice(0, -".md".length), source))));
424
+ });
425
+ // src/score.ts
426
+ import { Effect as Effect6, FileSystem as FileSystem2, Path as Path2, Result as Result3, Runtime as Runtime4, Schema as Schema5, SchemaIssue as SchemaIssue2 } from "effect";
427
+ var Range = Schema5.Tuple([Schema5.Int, Schema5.Int]).check(Schema5.makeFilter(([start, end]) => 1 <= start && start <= end || "lines must be [start, end] with 1 <= start <= end"));
428
+ var LabelFields = Schema5.Struct({
429
+ path: Schema5.optional(Schema5.NonEmptyString),
430
+ item: Schema5.optional(Schema5.NonEmptyString),
431
+ label: Schema5.Literals(["violates", "complies", "ambiguous"]),
432
+ lines: Schema5.optional(Schema5.Array(Range)),
433
+ split: Schema5.Literals(["dev", "test"]),
434
+ note: Schema5.optional(Schema5.String)
435
+ });
436
+
437
+ class LabelError extends Schema5.TaggedError()("LabelError", {
438
+ message: Schema5.String
439
+ }) {
440
+ [Runtime4.errorExitCode] = 2;
441
+ }
442
+ var formatIssues2 = SchemaIssue2.makeFormatterStandardSchemaV1();
443
+ var targetKey = (record) => record.path !== undefined ? record.path : `item ${record.item}`;
444
+ var parseLabels = Effect6.fn("parseLabels")(function* (name, text) {
445
+ const records = [];
446
+ const seen = new Set;
447
+ for (const [index, line] of text.split(`
448
+ `).entries()) {
449
+ if (line.trim() === "")
450
+ continue;
451
+ const fail = (reason) => new LabelError({ message: `${name}:${index + 1}: ${reason}` });
452
+ const value = yield* Effect6.try({ try: () => JSON.parse(line), catch: () => fail("not valid JSON") });
453
+ if (typeof value !== "object" || value === null || Array.isArray(value))
454
+ return yield* fail("not an object");
455
+ const unknown = Object.keys(value).filter((key) => !(key in LabelFields.fields));
456
+ if (unknown.length > 0)
457
+ return yield* fail(`unknown field ${unknown.join(", ")}`);
458
+ const record = yield* Schema5.decodeUnknownEffect(LabelFields)(value).pipe(Effect6.mapError((error) => fail(formatIssues2(error.issue).issues.map(({ path, message }) => `${path?.length ? `${path.join(".")}: ` : ""}${message}`).join("; "))));
459
+ if (record.path === undefined === (record.item === undefined))
460
+ return yield* fail("give exactly one of path and item");
461
+ if (record.label === "violates" && record.path !== undefined && (record.lines ?? []).length === 0) {
462
+ return yield* fail("a violating file needs lines: the ranges of its violations");
463
+ }
464
+ if (record.lines !== undefined && (record.label !== "violates" || record.path === undefined)) {
465
+ return yield* fail("lines belong only to a violating file");
466
+ }
467
+ const key = targetKey(record);
468
+ if (seen.has(key))
469
+ return yield* fail(`${key} is labeled twice`);
470
+ seen.add(key);
471
+ records.push(record);
472
+ }
473
+ return records;
474
+ });
475
+ var loadLabels = Effect6.fn("loadLabels")(function* (policyName, directory = ".lint-anything/labels") {
476
+ const fs = yield* FileSystem2.FileSystem;
477
+ const path = yield* Path2.Path;
478
+ const file = path.join(directory, `${policyName}.jsonl`);
479
+ const text = yield* fs.readFileString(file).pipe(Effect6.mapError(() => new LabelError({ message: `cannot read ${file}; label the policy's code there first` })));
480
+ return yield* parseLabels(file, text);
481
+ });
482
+ var overlaps = (lines, [start, end]) => lines[0] <= end && start <= lines[1];
483
+ var itemLabel = (target, item) => target.label !== "violates" || target.lines === undefined ? target.label : target.lines.some((range) => item.lines === undefined || overlaps(item.lines, range)) ? "violates" : "complies";
484
+ var reachedRanges = (ranges, items) => ranges.filter((range) => items.some((item) => item.lines === undefined || overlaps(item.lines, range))).length;
485
+ var targetScore = (scores) => {
486
+ const answered = scores.filter((score) => score !== null);
487
+ return scores.length > 0 && answered.length === 0 ? null : Math.max(0, ...answered);
488
+ };
489
+ var itemsOf = (record, items) => items.filter((item) => record.path !== undefined ? item.path === record.path : item.path === undefined && item.id === record.item);
490
+ var cliThresholds = { floor: reviewFloor, threshold: defaultThreshold };
491
+ var verdict = (score, { floor, threshold }) => score <= floor ? "pass" : score >= threshold ? "violation" : "review";
492
+ var idealScore = (scored) => ({ complies: 0, ambiguous: 0.5, violates: 1 })[scored.label];
493
+ var probability = (scored) => scored.score ?? 0;
494
+ var separation = (low, high) => {
495
+ if (low.length === 0 || high.length === 0)
496
+ return Number.NaN;
497
+ const all = [...low.map((value) => ({ value, high: false })), ...high.map((value) => ({ value, high: true }))].sort((a, b) => a.value - b.value);
498
+ let rankSum = 0;
499
+ for (let start = 0;start < all.length; ) {
500
+ let end = start;
501
+ while (end < all.length && all[end].value === all[start].value)
502
+ end++;
503
+ const meanRank = (start + 1 + end) / 2;
504
+ for (let index = start;index < end; index++)
505
+ if (all[index].high)
506
+ rankSum += meanRank;
507
+ start = end;
508
+ }
509
+ return (rankSum - high.length * (high.length + 1) / 2) / (high.length * low.length);
510
+ };
511
+ var scoresOf = (cases, label) => cases.filter((scored) => scored.label === label).map(probability);
512
+ var auroc = (cases) => separation(scoresOf(cases, "complies"), scoresOf(cases, "violates"));
513
+ var concordance = (cases) => {
514
+ const clean = scoresOf(cases, "complies");
515
+ const ambiguous = scoresOf(cases, "ambiguous");
516
+ const violates = scoresOf(cases, "violates");
517
+ const parts = [[clean, ambiguous], [clean, violates], [ambiguous, violates]];
518
+ let weighted = 0;
519
+ let pairs = 0;
520
+ for (const [low, high] of parts) {
521
+ const count = low.length * high.length;
522
+ if (count === 0)
523
+ continue;
524
+ weighted += separation(low, high) * count;
525
+ pairs += count;
526
+ }
527
+ return pairs === 0 ? Number.NaN : weighted / pairs;
528
+ };
529
+ var brier = (cases) => cases.reduce((sum, scored) => sum + (idealScore(scored) - probability(scored)) ** 2, 0) / cases.length;
530
+ var share = (cases, include, hit) => {
531
+ const included = cases.filter(include);
532
+ return included.length === 0 ? Number.NaN : included.filter(hit).length / included.length;
533
+ };
534
+ var verdictF1 = (cases, thresholds) => {
535
+ const pairs = [["complies", "pass"], ["ambiguous", "review"], ["violates", "violation"]];
536
+ const f1s = pairs.flatMap(([label, predicted]) => {
537
+ const actual = cases.filter((scored) => scored.label === label).length;
538
+ const flagged = cases.filter((scored) => verdict(probability(scored), thresholds) === predicted);
539
+ if (actual === 0 && flagged.length === 0)
540
+ return [];
541
+ const hits = flagged.filter((scored) => scored.label === label).length;
542
+ return [2 * hits / (actual + flagged.length)];
543
+ });
544
+ return f1s.reduce((sum, value) => sum + value, 0) / f1s.length;
545
+ };
546
+ var tuneThresholds = (cases) => {
547
+ let best = cliThresholds;
548
+ let bestF1 = -1;
549
+ for (let floor = 0.025;floor < 1; floor += 0.025) {
550
+ for (let threshold = floor + 0.025;threshold <= 1; threshold += 0.025) {
551
+ const candidate = { floor: Number(floor.toFixed(3)), threshold: Number(threshold.toFixed(3)) };
552
+ const f1 = verdictF1(cases, candidate);
553
+ if (f1 > bestF1) {
554
+ bestF1 = f1;
555
+ best = candidate;
556
+ }
557
+ }
558
+ }
559
+ return best;
560
+ };
561
+ var quality = (cases, thresholds) => {
562
+ const of = (scored) => verdict(probability(scored), thresholds);
563
+ return {
564
+ auroc: auroc(cases),
565
+ concordance: concordance(cases),
566
+ brier: brier(cases),
567
+ f1: verdictF1(cases, thresholds),
568
+ caught: share(cases, (s) => s.label === "violates", (s) => of(s) === "violation"),
569
+ surfaced: share(cases, (s) => s.label === "violates", (s) => of(s) !== "pass"),
570
+ ambiguousToReview: share(cases, (s) => s.label === "ambiguous", (s) => of(s) === "review"),
571
+ falseAlarms: share(cases, (s) => s.label === "complies", (s) => of(s) !== "pass"),
572
+ inconclusive: cases.filter((scored) => scored.score === null).length
573
+ };
574
+ };
575
+ var caught = (cases) => quality(cases, cliThresholds).caught;
576
+ var falseAlarms = (cases) => quality(cases, cliThresholds).falseAlarms;
577
+ var mulberry32 = (seed) => () => {
578
+ seed = seed + 1831565813 | 0;
579
+ let t = Math.imul(seed ^ seed >>> 15, 1 | seed);
580
+ t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
581
+ return ((t ^ t >>> 14) >>> 0) / 4294967296;
582
+ };
583
+ var byGroup = (cases) => {
584
+ const groups = new Map;
585
+ for (const scored of cases) {
586
+ const list = groups.get(scored.group) ?? [];
587
+ list.push(scored);
588
+ groups.set(scored.group, list);
589
+ }
590
+ return groups;
591
+ };
592
+ var percentiles = (values) => {
593
+ values.sort((a, b) => a - b);
594
+ return values.length === 0 ? [Number.NaN, Number.NaN] : [values[Math.floor(values.length * 0.025)], values[Math.floor(values.length * 0.975)]];
595
+ };
596
+ var pairedInterval = (baseline, candidate, metric, samples = 2000) => {
597
+ const before = byGroup(baseline);
598
+ const after = byGroup(candidate);
599
+ const groups = [...before.keys()].sort();
600
+ const random = mulberry32(24301);
601
+ const deltas = [];
602
+ for (let sample = 0;sample < samples; sample++) {
603
+ const left = [];
604
+ const right = [];
605
+ for (let draw = 0;draw < groups.length; draw++) {
606
+ const group = groups[Math.floor(random() * groups.length)];
607
+ left.push(...before.get(group));
608
+ right.push(...after.get(group) ?? []);
609
+ }
610
+ const delta = metric(right) - metric(left);
611
+ if (!Number.isNaN(delta))
612
+ deltas.push(delta);
613
+ }
614
+ return percentiles(deltas);
615
+ };
616
+ var bootstrapInterval = (cases, metric, samples = 2000) => {
617
+ const groups = byGroup(cases);
618
+ const keys = [...groups.keys()].sort();
619
+ const random = mulberry32(24301);
620
+ const values = [];
621
+ for (let sample = 0;sample < samples; sample++) {
622
+ const drawn = [];
623
+ for (let draw = 0;draw < keys.length; draw++)
624
+ drawn.push(...groups.get(keys[Math.floor(random() * keys.length)]));
625
+ const value = metric(drawn);
626
+ if (!Number.isNaN(value))
627
+ values.push(value);
628
+ }
629
+ return percentiles(values);
630
+ };
631
+ var scorePolicy = Effect6.fn("scorePolicy")(function* (policy, labels, cwd) {
632
+ const selected = yield* select(policy, cwd);
633
+ const ofTarget = labels.map((record) => ({ record, items: itemsOf(record, selected) }));
634
+ const inTargets = new Set(ofTarget.flatMap(({ items }) => items));
635
+ const judged = selected.filter((item) => inTargets.has(item));
636
+ const answers = yield* judge(policy, judged);
637
+ const scoreOf = new Map(judged.map((item, index) => [item, answers[index]]));
638
+ const score = (item) => Result3.match(scoreOf.get(item), { onSuccess: (violates) => violates, onFailure: () => null });
639
+ const reach = { dev: { reached: 0, total: 0 }, test: { reached: 0, total: 0 } };
640
+ for (const { record, items } of ofTarget) {
641
+ const ranges = record.label === "violates" ? record.lines ?? [] : [];
642
+ reach[record.split].reached += reachedRanges(ranges, items);
643
+ reach[record.split].total += ranges.length;
644
+ }
645
+ return {
646
+ targets: ofTarget.map(({ record, items }) => ({ subject: record, label: record.label, group: targetKey(record), score: targetScore(items.map(score)) })),
647
+ items: ofTarget.flatMap(({ record, items }) => items.map((item) => ({ subject: item, label: itemLabel(record, item), group: targetKey(record), score: score(item) }))),
648
+ reach
649
+ };
650
+ });
651
+ export {
652
+ Clef,
653
+ ClefError,
654
+ LabelError,
655
+ PolicyError,
656
+ RequestTooLarge,
657
+ SelectorError,
658
+ aggregate,
659
+ auroc,
660
+ bootstrapInterval,
661
+ brier,
662
+ cacheMaxAgeMs,
663
+ caught,
664
+ cliThresholds,
665
+ concordance,
666
+ decide,
667
+ defaultThreshold,
668
+ falseAlarms,
669
+ inputTokenPrice,
670
+ itemLabel,
671
+ itemQuestion,
672
+ itemsOf,
673
+ judge,
674
+ killRunningSelectors,
675
+ lint,
676
+ loadLabels,
677
+ loadPolicies,
678
+ model,
679
+ noUsage,
680
+ pairedInterval,
681
+ parseItem,
682
+ parseLabels,
683
+ parsePolicy,
684
+ planRequests,
685
+ quality,
686
+ reachedRanges,
687
+ reviewFloor,
688
+ runSelector,
689
+ scorePolicy,
690
+ select,
691
+ selectorTimeoutMinutes,
692
+ sumUsage,
693
+ targetKey,
694
+ targetScore,
695
+ tuneThresholds,
696
+ verdict,
697
+ verdictF1,
698
+ verdictOf
699
+ };