@andrueandersoncs/lint-anything 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/lib.js CHANGED
@@ -29,8 +29,13 @@ var maxImageBytes = 4 * 1024 * 1024;
29
29
  var maxMediaBytes = 8 * 1024 * 1024;
30
30
  var readMedium = async (path) => {
31
31
  const { type, kind } = mediaTypeOf(path);
32
- const data = await Bun.file(path).bytes();
33
- return { path, type, kind, bytes: data.length, digest: new Bun.CryptoHasher("sha256").update(data).digest("hex") };
32
+ const hasher = new Bun.CryptoHasher("sha256");
33
+ let bytes = 0;
34
+ for await (const chunk of Bun.file(path).stream()) {
35
+ hasher.update(chunk);
36
+ bytes += chunk.byteLength;
37
+ }
38
+ return { path, type, kind, bytes, digest: hasher.digest("hex") };
34
39
  };
35
40
  var mediaBase64 = async (medium) => Buffer.from(await Bun.file(medium.path).bytes()).toString("base64");
36
41
 
@@ -223,16 +228,56 @@ var choice = Effect.gen(function* () {
223
228
  }
224
229
  return { kind: "workers-ai", model: name };
225
230
  });
231
+ var secondaryChoice = Effect.gen(function* () {
232
+ const endpoint = yield* setting("LINT_ANYTHING_SECONDARY_ENDPOINT");
233
+ const model = yield* setting("LINT_ANYTHING_SECONDARY_MODEL");
234
+ const apiKey = yield* setting("LINT_ANYTHING_SECONDARY_API_KEY");
235
+ if (Option.isSome(endpoint)) {
236
+ if (!isHttpUrl(endpoint.value)) {
237
+ return yield* configError(`LINT_ANYTHING_SECONDARY_ENDPOINT must be an http or https URL, such as http://localhost:4000/v1/systemone`);
238
+ }
239
+ if (Option.isNone(model)) {
240
+ return yield* configError(`LINT_ANYTHING_SECONDARY_MODEL is not set; LINT_ANYTHING_SECONDARY_ENDPOINT needs the name of the model to ask`);
241
+ }
242
+ return Option.some({
243
+ kind: "endpoint",
244
+ url: endpoint.value,
245
+ model: model.value,
246
+ token: Option.getOrUndefined(Option.map(apiKey, Redacted.make))
247
+ });
248
+ }
249
+ if (Option.isSome(apiKey)) {
250
+ return yield* configError("LINT_ANYTHING_SECONDARY_API_KEY is set but LINT_ANYTHING_SECONDARY_ENDPOINT is not; set the endpoint the key is for (Clef on Workers AI uses CLOUDFLARE_AUTH_TOKEN)");
251
+ }
252
+ if (Option.isNone(model))
253
+ return Option.none();
254
+ const name = model.value;
255
+ if (!/^[\w.-]+$/.test(name)) {
256
+ return yield* configError(`LINT_ANYTHING_SECONDARY_MODEL ${name} is not a Cloudflare model; to ask it, set LINT_ANYTHING_SECONDARY_ENDPOINT to the decision endpoint that serves it`);
257
+ }
258
+ return Option.some({ kind: "workers-ai", model: name });
259
+ });
260
+ var resolveWorkersAi = (model) => Effect.gen(function* () {
261
+ const required = (name, config) => config.pipe(Effect.mapError(() => configError(`${name} is not set; set it in the environment or in .env`)));
262
+ const accountId = yield* required("CLOUDFLARE_ACCOUNT_ID", Config.String("CLOUDFLARE_ACCOUNT_ID"));
263
+ const token = yield* required("CLOUDFLARE_AUTH_TOKEN", Config.Redacted("CLOUDFLARE_AUTH_TOKEN"));
264
+ const baseUrl = yield* Config.String("CLOUDFLARE_API_BASE_URL").pipe(Config.withDefault(defaultCloudflareBaseUrl), Effect.mapError((error) => configError(`CLOUDFLARE_API_BASE_URL: ${error.message}`)));
265
+ return workersAiTarget({ accountId, token, model, baseUrl });
266
+ });
226
267
  var configuredModel = Effect.map(choice, (chosen) => chosen.kind === "endpoint" ? endpointModel(chosen.url, chosen.model) : workersAiModel(chosen.model));
227
268
  var configuredTarget = Effect.gen(function* () {
228
269
  const chosen = yield* choice;
229
270
  if (chosen.kind === "endpoint")
230
271
  return endpointTarget(chosen);
231
- const required = (name, config) => config.pipe(Effect.mapError(() => configError(`${name} is not set; set it in the environment or in .env`)));
232
- const accountId = yield* required("CLOUDFLARE_ACCOUNT_ID", Config.String("CLOUDFLARE_ACCOUNT_ID"));
233
- const token = yield* required("CLOUDFLARE_AUTH_TOKEN", Config.Redacted("CLOUDFLARE_AUTH_TOKEN"));
234
- const baseUrl = yield* Config.String("CLOUDFLARE_API_BASE_URL").pipe(Config.withDefault(defaultCloudflareBaseUrl), Effect.mapError((error) => configError(`CLOUDFLARE_API_BASE_URL: ${error.message}`)));
235
- return workersAiTarget({ accountId, token, model: chosen.model, baseUrl });
272
+ return yield* resolveWorkersAi(chosen.model);
273
+ });
274
+ var configuredSecondaryTarget = Effect.gen(function* () {
275
+ const chosen = yield* secondaryChoice;
276
+ if (Option.isNone(chosen))
277
+ return Option.none();
278
+ if (chosen.value.kind === "endpoint")
279
+ return Option.some(endpointTarget(chosen.value));
280
+ return Option.some(yield* resolveWorkersAi(chosen.value.model));
236
281
  });
237
282
  var pack = (model, state, media, values, question) => {
238
283
  const wire = wires[model.format];
@@ -353,15 +398,83 @@ var failureMessage = (name, error) => {
353
398
  const detail = (error instanceof Error ? error.message : String(error)).replace(/\s*\n\s*/g, " ");
354
399
  return Effect.succeed(`${name} sent an unexpected response: ${detail}`);
355
400
  };
401
+ var jsonRetryDelayMs = (text) => {
402
+ const match = /"retryDelay":\s*"(\d+(?:\.\d+)?)s"/.exec(text);
403
+ return match === null ? undefined : Math.ceil(Number(match[1]) * 1000);
404
+ };
356
405
 
357
406
  class DecisionModel extends Context.Service()("lint-anything/DecisionModel") {
358
- static layer = (cacheDirectory) => Layer.effect(DecisionModel, Effect.flatMap(configuredTarget, (target) => make(target, cacheDirectory))).pipe(Layer.provide(FetchHttpClient.layer));
407
+ static layer = (cacheDirectory) => Layer.effect(DecisionModel, Effect.gen(function* () {
408
+ const primary = yield* configuredTarget;
409
+ const secondary = yield* configuredSecondaryTarget;
410
+ if (Option.isNone(secondary))
411
+ return yield* make(primary, cacheDirectory);
412
+ if (secondary.value.cacheKey === primary.cacheKey) {
413
+ return yield* configError(`secondary decision model (${secondary.value.name}) is the same as the primary model; use a different model for cascading`);
414
+ }
415
+ return yield* makeCascade(primary, secondary.value, cacheDirectory);
416
+ })).pipe(Layer.provide(FetchHttpClient.layer));
359
417
  static layerFor = (target, cacheDirectory) => Layer.effect(DecisionModel, make(target, cacheDirectory)).pipe(Layer.provide(FetchHttpClient.layer));
418
+ static cascadeLayer = (primary, secondary, cacheDirectory, options) => Layer.effect(DecisionModel, makeCascade(primary, secondary, cacheDirectory, options)).pipe(Layer.provide(FetchHttpClient.layer));
360
419
  }
420
+ var makeCascade = Effect.fnUntraced(function* (primaryTarget, secondaryTarget, cacheDirectory, options) {
421
+ const low = options?.low ?? 0.1;
422
+ const high = options?.high ?? 0.9;
423
+ const combine = options?.combine ?? ((a, b) => (a + b) / 2);
424
+ const primary = yield* make(primaryTarget, cacheDirectory);
425
+ const secondary = yield* make(secondaryTarget, cacheDirectory);
426
+ return DecisionModel.of({
427
+ target: {
428
+ ...primary.target,
429
+ name: `${primary.target.name}+${secondary.target.name}`,
430
+ cacheKey: `${primary.target.cacheKey}+${secondary.target.cacheKey}`,
431
+ inputTokenPrice: undefined
432
+ },
433
+ ask: Effect.fn("DecisionModel.cascade.ask")(function* (source, state, questions) {
434
+ const first = [...yield* primary.ask(source, state, questions)];
435
+ const escalate = [];
436
+ first.forEach((result, index) => {
437
+ if (Result.isSuccess(result) && result.success >= low && result.success <= high) {
438
+ escalate.push(index);
439
+ }
440
+ });
441
+ if (escalate.length === 0)
442
+ return first;
443
+ const second = yield* secondary.ask(source, state, escalate.map((index) => questions[index]));
444
+ escalate.forEach((origIndex, subIndex) => {
445
+ const a = first[origIndex];
446
+ const b = second[subIndex];
447
+ if (Result.isSuccess(a) && Result.isSuccess(b)) {
448
+ first[origIndex] = Result.succeed(combine(a.success, b.success));
449
+ }
450
+ });
451
+ return first;
452
+ }),
453
+ stats: Effect.map(Effect.all([primary.stats, secondary.stats]), ([a, b]) => {
454
+ const merged = new Map(a);
455
+ for (const [source, usage] of b) {
456
+ merged.set(source, sumUsage([merged.get(source) ?? noUsage, usage]));
457
+ }
458
+ return merged;
459
+ })
460
+ });
461
+ });
361
462
  var make = Effect.fnUntraced(function* (target, cacheDirectory) {
362
463
  const wire = wires[target.format];
363
464
  const { token } = target;
364
- const client = (yield* HttpClient.HttpClient).pipe(HttpClient.mapRequest(flow(token === undefined ? (request) => request : HttpClientRequest.setHeader(...wire.keyHeader(Redacted.value(token))), HttpClientRequest.acceptJson)), HttpClient.filterStatusOk, HttpClient.transformResponse(flow(Effect.tap((response) => response.text), Effect.timeout(attemptTimeout))), HttpClient.retryTransient({ schedule: Schedule.exponential(500), times: attemptRetries }));
465
+ const client = (yield* HttpClient.HttpClient).pipe(HttpClient.mapRequest(flow(token === undefined ? (request) => request : HttpClientRequest.setHeader(...wire.keyHeader(Redacted.value(token))), HttpClientRequest.acceptJson)), HttpClient.filterStatusOk, HttpClient.transformResponse(flow(Effect.tap((response) => response.text), Effect.timeout(attemptTimeout), Effect.tapError((error) => {
466
+ if (!HttpClientError.isHttpClientError(error) || error.reason._tag !== "StatusCodeError")
467
+ return Effect.void;
468
+ const { response } = error.reason;
469
+ if (response.status !== 429 || response.headers["retry-after"] !== undefined)
470
+ return Effect.void;
471
+ return response.text.pipe(Effect.timeout("5 seconds"), Effect.flatMap((text) => {
472
+ const delayMs = jsonRetryDelayMs(text);
473
+ return delayMs !== undefined && delayMs > 0 ? Effect.sleep(delayMs) : Effect.void;
474
+ }), Effect.orElseSucceed(() => {
475
+ return;
476
+ }));
477
+ }))), HttpClient.retryTransient({ schedule: Schedule.exponential(500), times: attemptRetries }));
365
478
  const cache = cacheDirectory === undefined ? undefined : yield* Effect.acquireRelease(Effect.try({
366
479
  try: () => openAnswerCache(cacheDirectory, Date.now(), target),
367
480
  catch: (error) => new DecisionModelError({
@@ -453,21 +566,71 @@ var make = Effect.fnUntraced(function* (target, cacheDirectory) {
453
566
  else
454
567
  results[index] = Result.fail(tooMuch);
455
568
  }
456
- const { requests, oversized } = planRequests(state, sendable.map((index) => questions[index]), target);
457
- for (const { value, bytes } of oversized)
458
- results[sendable[value]] = Result.fail(tooLarge(bytes));
569
+ const uniqueSendable = [];
570
+ const duplicates = new Map;
571
+ const byFingerprint = new Map;
572
+ for (const index of sendable) {
573
+ const fingerprint = `${JSON.stringify(noul(questions[index]))}\x00${prints[index]}`;
574
+ const leader = byFingerprint.get(fingerprint);
575
+ if (leader === undefined) {
576
+ const slot = uniqueSendable.length;
577
+ byFingerprint.set(fingerprint, slot);
578
+ uniqueSendable.push(index);
579
+ duplicates.set(slot, [index]);
580
+ } else {
581
+ duplicates.get(leader).push(index);
582
+ }
583
+ }
584
+ const minWarmBatch = 5;
585
+ const paddingIndices = [];
586
+ if (target.format === "system-one" && uniqueSendable.length > 0 && uniqueSendable.length < minWarmBatch && uniqueSendable.every((index) => (questions[index].media?.length ?? 0) === 0)) {
587
+ const siblingPool = [];
588
+ for (let i = 0;i < questions.length; i++) {
589
+ if (cached[i] !== undefined && (questions[i].media?.length ?? 0) === 0 && questions[i].instructions.length <= 4000) {
590
+ siblingPool.push(i);
591
+ }
592
+ }
593
+ const needed = minWarmBatch - uniqueSendable.length;
594
+ if (siblingPool.length > 0) {
595
+ const step = Math.max(1, Math.floor(siblingPool.length / needed));
596
+ for (let i = 0;i < siblingPool.length && paddingIndices.length < needed; i += step) {
597
+ paddingIndices.push(siblingPool[i]);
598
+ }
599
+ }
600
+ }
601
+ let paddingOffset = paddingIndices.length;
602
+ let planned = planRequests(state, [...paddingIndices.map((index) => questions[index]), ...uniqueSendable.map((index) => questions[index])], target);
603
+ if (paddingOffset > 0 && planned.requests.length > 1) {
604
+ paddingOffset = 0;
605
+ planned = planRequests(state, uniqueSendable.map((index) => questions[index]), target);
606
+ }
607
+ const { requests, oversized } = planned;
608
+ for (const { value, bytes } of oversized) {
609
+ if (value < paddingOffset)
610
+ continue;
611
+ const failure = Result.fail(tooLarge(bytes));
612
+ for (const index of duplicates.get(value - paddingOffset))
613
+ results[index] = failure;
614
+ }
459
615
  yield* Effect.forEach(requests, (request) => send(source, state, request, media).pipe(Effect.tap((received) => {
460
616
  const answered = [];
461
617
  for (const { value, answer } of received) {
462
- const index = sendable[value];
618
+ if (value < paddingOffset)
619
+ continue;
620
+ const targets = duplicates.get(value - paddingOffset);
621
+ const leader = targets[0];
463
622
  if (answer === "declined") {
464
- results[index] = Result.fail(declined);
623
+ const failure = Result.fail(declined);
624
+ for (const index of targets)
625
+ results[index] = failure;
465
626
  } else {
466
- results[index] = Result.succeed(answer);
467
- answered.push({ question: questions[index], print: prints[index], noul: answer });
627
+ const success = Result.succeed(answer);
628
+ for (const index of targets)
629
+ results[index] = success;
630
+ answered.push({ question: questions[leader], print: prints[leader], noul: answer });
468
631
  }
469
632
  }
470
- return cache === undefined ? Effect.void : cache.set(state, answered);
633
+ return cache === undefined || answered.length === 0 ? Effect.void : cache.set(state, answered);
471
634
  })), { concurrency: "unbounded", discard: true });
472
635
  return results;
473
636
  });
@@ -619,7 +782,8 @@ var itemLocation = (item) => ({
619
782
  ...item.path === undefined ? {} : { path: item.path },
620
783
  ...item.lines === undefined ? {} : { lines: item.lines }
621
784
  });
622
- var byLocation = (a, b) => (a.path ?? "").localeCompare(b.path ?? "") || (a.lines?.[0] ?? 0) - (b.lines?.[0] ?? 0) || a.id.localeCompare(b.id);
785
+ var compareStrings = (a, b) => a < b ? -1 : a > b ? 1 : 0;
786
+ var byLocation = (a, b) => compareStrings(a.path ?? "", b.path ?? "") || (a.lines?.[0] ?? 0) - (b.lines?.[0] ?? 0) || compareStrings(a.id, b.id);
623
787
  var select = Effect4.fn("select")(function* (policy, cwd, runs) {
624
788
  return (yield* runSelector(policy, cwd, runs)).toSorted(byLocation);
625
789
  });
@@ -754,7 +918,35 @@ var loadLabels = Effect6.fn("loadLabels")(function* (policyName, directory = ".l
754
918
  return yield* parseLabels(file, text);
755
919
  });
756
920
  var overlaps = (lines, [start, end]) => lines[0] <= end && start <= lines[1];
921
+ var overlapLength = ([a0, a1], [b0, b1]) => Math.max(0, Math.min(a1, b1) - Math.max(a0, b0) + 1);
757
922
  var itemLabel = (target, item) => target.label !== "violates" || target.lines === undefined ? target.label : target.lines.some((range) => item.lines === undefined || overlaps(item.lines, range)) ? "violates" : "complies";
923
+ var labelFileItems = (target, items) => {
924
+ if (target.label !== "violates" || target.lines === undefined)
925
+ return items.map(() => target.label);
926
+ const ranges = target.lines;
927
+ return items.map((item) => {
928
+ if (item.lines === undefined)
929
+ return "violates";
930
+ const [start, end] = item.lines;
931
+ const matched = ranges.filter((range) => overlaps(item.lines, range));
932
+ if (matched.length === 0)
933
+ return "complies";
934
+ const onlyNested = matched.every((range) => {
935
+ const rangeLen = range[1] - range[0] + 1;
936
+ const mine = overlapLength(item.lines, range);
937
+ if (mine / rangeLen >= 0.5)
938
+ return false;
939
+ return items.some((other) => {
940
+ if (other === item || other.lines === undefined)
941
+ return false;
942
+ const [os, oe] = other.lines;
943
+ const strictlyEncloses = os <= start && end <= oe && (os < start || oe > end);
944
+ return strictlyEncloses && overlapLength(other.lines, range) > mine;
945
+ });
946
+ });
947
+ return onlyNested ? "complies" : "violates";
948
+ });
949
+ };
758
950
  var reachedRanges = (ranges, items) => ranges.filter((range) => items.some((item) => item.lines === undefined || overlaps(item.lines, range))).length;
759
951
  var targetScore = (scores) => {
760
952
  const answered = scores.filter((score) => score !== null);
@@ -784,6 +976,55 @@ var separation = (low, high) => {
784
976
  };
785
977
  var scoresOf = (cases, label) => cases.filter((scored) => scored.label === label).map(probability);
786
978
  var auroc = (cases) => separation(scoresOf(cases, "complies"), scoresOf(cases, "violates"));
979
+ var auprc = (cases) => {
980
+ const clear = cases.filter((scored) => scored.label !== "ambiguous");
981
+ const totalPos = clear.filter((scored) => scored.label === "violates").length;
982
+ if (totalPos === 0 || clear.length === totalPos)
983
+ return Number.NaN;
984
+ const sorted = clear.map((scored) => ({ score: probability(scored), pos: scored.label === "violates" })).sort((a, b) => b.score - a.score);
985
+ let tp = 0;
986
+ let seen = 0;
987
+ let ap = 0;
988
+ for (let start = 0;start < sorted.length; ) {
989
+ let end = start;
990
+ let groupPos = 0;
991
+ while (end < sorted.length && sorted[end].score === sorted[start].score) {
992
+ if (sorted[end].pos)
993
+ groupPos++;
994
+ end++;
995
+ }
996
+ const groupSize = end - start;
997
+ if (groupPos > 0) {
998
+ let sumPrec = 0;
999
+ for (let k = 1;k <= groupPos; k++) {
1000
+ const expectedSeen = seen + k * (groupSize + 1) / (groupPos + 1);
1001
+ sumPrec += (tp + k) / expectedSeen;
1002
+ }
1003
+ ap += sumPrec / totalPos;
1004
+ }
1005
+ tp += groupPos;
1006
+ seen += groupSize;
1007
+ start = end;
1008
+ }
1009
+ return ap;
1010
+ };
1011
+ var tprAtFpr = (cases, fpr) => {
1012
+ const clean = scoresOf(cases, "complies").sort((a, b) => b - a);
1013
+ const viol = scoresOf(cases, "violates");
1014
+ if (clean.length === 0 || viol.length === 0)
1015
+ return Number.NaN;
1016
+ const cut = clean[Math.min(clean.length - 1, Math.floor(clean.length * fpr))];
1017
+ const above = viol.filter((value) => value > cut).length;
1018
+ const tied = viol.filter((value) => value === cut).length;
1019
+ const cleanAbove = clean.filter((value) => value > cut).length;
1020
+ const cleanTied = clean.filter((value) => value === cut).length;
1021
+ const allowedFromTie = cleanTied > 0 ? Math.max(0, Math.min(1, (fpr * clean.length - cleanAbove) / cleanTied)) : 0;
1022
+ return (above + tied * allowedFromTie) / viol.length;
1023
+ };
1024
+ var violationPrecision = (cases, thresholds = cliThresholds) => {
1025
+ const clearFlagged = cases.filter((scored) => scored.label !== "ambiguous" && verdict(probability(scored), thresholds) === "violation");
1026
+ return clearFlagged.length === 0 ? Number.NaN : clearFlagged.filter((scored) => scored.label === "violates").length / clearFlagged.length;
1027
+ };
787
1028
  var concordance = (cases) => {
788
1029
  const clean = scoresOf(cases, "complies");
789
1030
  const ambiguous = scoresOf(cases, "ambiguous");
@@ -918,7 +1159,10 @@ var scorePolicy = Effect6.fn("scorePolicy")(function* (policy, labels, cwd) {
918
1159
  }
919
1160
  return {
920
1161
  targets: ofTarget.map(({ record, items }) => ({ subject: record, label: record.label, group: targetKey(record), score: targetScore(items.map(score)) })),
921
- items: ofTarget.flatMap(({ record, items }) => items.map((item) => ({ subject: item, label: itemLabel(record, item), group: targetKey(record), score: score(item) }))),
1162
+ items: ofTarget.flatMap(({ record, items }) => {
1163
+ const itemLabels = labelFileItems(record, items);
1164
+ return items.map((item, index) => ({ subject: item, label: itemLabels[index], group: targetKey(record), score: score(item) }));
1165
+ }),
922
1166
  reach
923
1167
  };
924
1168
  });
@@ -936,6 +1180,7 @@ export {
936
1180
  PolicyError,
937
1181
  RequestTooLarge,
938
1182
  SelectorError,
1183
+ auprc,
939
1184
  auroc,
940
1185
  bootstrapInterval,
941
1186
  brier,
@@ -946,6 +1191,7 @@ export {
946
1191
  cliThresholds,
947
1192
  concordance,
948
1193
  configuredModel,
1194
+ configuredSecondaryTarget,
949
1195
  configuredTarget,
950
1196
  decide,
951
1197
  declinesAll,
@@ -959,6 +1205,7 @@ export {
959
1205
  judge,
960
1206
  keyHeader,
961
1207
  killRunningSelectors,
1208
+ labelFileItems,
962
1209
  lint,
963
1210
  loadLabels,
964
1211
  loadPolicies,
@@ -978,9 +1225,11 @@ export {
978
1225
  sumUsage,
979
1226
  targetKey,
980
1227
  targetScore,
1228
+ tprAtFpr,
981
1229
  tuneThresholds,
982
1230
  verdict,
983
1231
  verdictF1,
984
1232
  verdictOf,
1233
+ violationPrecision,
985
1234
  workersAiTarget
986
1235
  };
@@ -1,4 +1,4 @@
1
- import { Cause, Context, Effect, Layer, Redacted, Result, Runtime, Schema } from "effect";
1
+ import { Cause, Context, Effect, Layer, Option, Redacted, Result, Runtime, Schema } from "effect";
2
2
  import { type MediaKind } from "./media.ts";
3
3
  /**
4
4
  * One yes/no question: `instructions` is what the decision model judges, and `criteria` say what a yes
@@ -138,6 +138,12 @@ export declare const configuredModel: Effect.Effect<Model, DecisionModelError>;
138
138
  * `CLOUDFLARE_AUTH_TOKEN`, and optionally `CLOUDFLARE_API_BASE_URL`.
139
139
  */
140
140
  export declare const configuredTarget: Effect.Effect<Target, DecisionModelError>;
141
+ /**
142
+ * The optional secondary decision model the environment configures for cascading (`LINT_ANYTHING_SECONDARY_ENDPOINT`,
143
+ * `LINT_ANYTHING_SECONDARY_MODEL`, `LINT_ANYTHING_SECONDARY_API_KEY`). Returns `Option.none()` when unset.
144
+ * Setting `LINT_ANYTHING_SECONDARY_MODEL` alone (for example `clef`) asks that Cloudflare model on Workers AI.
145
+ */
146
+ export declare const configuredSecondaryTarget: Effect.Effect<Option.Option<Target>, DecisionModelError>;
141
147
  /**
142
148
  * One request: its exact body, the media its questions show (by path), and what each question id answers.
143
149
  * The body of a request with media leaves them out; they are read, and put where its format takes them,
@@ -177,6 +183,15 @@ export interface Usage {
177
183
  }
178
184
  export declare const noUsage: Usage;
179
185
  export declare const sumUsage: (usages: Iterable<Usage>) => Usage;
186
+ /** Configuration for cascading two decision models: a fast primary and a secondary tiebreaker. */
187
+ export interface CascadeOptions {
188
+ /** Primary probabilities below `low` are accepted directly without calling `secondary` (default: 0.10). */
189
+ readonly low?: number;
190
+ /** Primary probabilities above `high` are accepted directly without calling `secondary` (default: 0.90). */
191
+ readonly high?: number;
192
+ /** Combines primary and secondary probabilities for items sent to both (default: arithmetic mean). */
193
+ readonly combine?: (primary: number, secondary: number) => number;
194
+ }
180
195
  declare const DecisionModel_base: Context.ServiceClass<DecisionModel, "lint-anything/DecisionModel", {
181
196
  /** The model asked, and where. */
182
197
  readonly target: Target;
@@ -190,12 +205,19 @@ declare const DecisionModel_base: Context.ServiceClass<DecisionModel, "lint-anyt
190
205
  }>;
191
206
  export declare class DecisionModel extends DecisionModel_base {
192
207
  /**
193
- * The decision model the environment configures (`configuredTarget`): Clef on Workers AI by default. With
194
- * `cacheDirectory`, answers are cached in a database there per question across runs, and answers older
195
- * than `cacheMaxAgeMs` are deleted and asked again.
208
+ * The decision model the environment configures (`configuredTarget`): Clef on Workers AI by default, or
209
+ * a cross-family cascade when `LINT_ANYTHING_SECONDARY_*` (`configuredSecondaryTarget`) is also set.
210
+ * With `cacheDirectory`, answers are cached in a database there per question across runs, and answers
211
+ * older than `cacheMaxAgeMs` are deleted and asked again.
196
212
  */
197
213
  static readonly layer: (cacheDirectory?: string) => Layer.Layer<DecisionModel, DecisionModelError, never>;
198
214
  /** `target`, whatever the environment says; answers are cached as with `layer`. */
199
215
  static readonly layerFor: (target: Target, cacheDirectory?: string) => Layer.Layer<DecisionModel, DecisionModelError, never>;
216
+ /**
217
+ * Cross-family cascade: asks `primary` first, and asks `secondary` only for questions whose primary
218
+ * probability lands in `[low, high]`, combining both probabilities with `combine` (default: mean).
219
+ * Combines `stats` from both models.
220
+ */
221
+ static readonly cascadeLayer: (primary: Target, secondary: Target, cacheDirectory?: string, options?: CascadeOptions) => Layer.Layer<DecisionModel, DecisionModelError, never>;
200
222
  }
201
223
  export {};
@@ -18,7 +18,7 @@
18
18
  * with the metrics and bootstrap intervals the evals use; the skill's score script is built on it.
19
19
  */
20
20
  import { DecisionModel, DecisionModelError } from "./decision-model.ts";
21
- export { cacheMaxAgeMs, clef, clefInputTokenPrice, configuredModel, configuredTarget, Declined, declinesAll, decodeReply, DecisionModel, DecisionModelError, endpointTarget, type Format, keyHeader, type Model, noUsage, planRequests, type Planned, type Question, RequestTooLarge, sumUsage, type Target, type Unanswered, type Usage, workersAiTarget } from "./decision-model.ts";
21
+ export { cacheMaxAgeMs, type CascadeOptions, clef, clefInputTokenPrice, configuredModel, configuredSecondaryTarget, configuredTarget, Declined, declinesAll, decodeReply, DecisionModel, DecisionModelError, endpointTarget, type Format, keyHeader, type Model, noUsage, planRequests, type Planned, type Question, RequestTooLarge, sumUsage, type Target, type Unanswered, type Usage, workersAiTarget } from "./decision-model.ts";
22
22
  /** @deprecated Renamed `DecisionModel`: it asks whichever decision model is configured, Clef by default. */
23
23
  export declare const Clef: typeof DecisionModel;
24
24
  /** @deprecated Renamed `DecisionModel`. */
@@ -31,4 +31,4 @@ export { type Item, parseItem } from "./item.ts";
31
31
  export { decide, defaultThreshold, type Finding, itemQuestion, judge, lint, reviewFloor, select, type Verdict, verdictOf } from "./lint.ts";
32
32
  export { loadPolicies, parsePolicy, type Policy, PolicyError } from "./policy.ts";
33
33
  export { killRunningSelectors, runSelector, SelectorError, type SelectorRuns, selectorTimeoutMinutes } from "./select.ts";
34
- export { auroc, bootstrapInterval, brier, caught, cliThresholds, concordance, falseAlarms, itemLabel, itemsOf, type Label, LabelError, type LabelRecord, loadLabels, pairedInterval, parseLabels, type PolicyScore, type Quality, quality, type Range, reachedRanges, type Scored, scorePolicy, type ScoreVerdict, type Split, targetKey, targetScore, type Thresholds, tuneThresholds, verdict, verdictF1 } from "./score.ts";
34
+ export { auprc, auroc, bootstrapInterval, brier, caught, cliThresholds, concordance, falseAlarms, itemLabel, itemsOf, type Label, LabelError, labelFileItems, type LabelRecord, loadLabels, pairedInterval, parseLabels, type PolicyScore, type Quality, quality, type Range, reachedRanges, type Scored, scorePolicy, type ScoreVerdict, type Split, targetKey, targetScore, type Thresholds, tprAtFpr, tuneThresholds, verdict, verdictF1, violationPrecision } from "./score.ts";
@@ -56,6 +56,16 @@ export declare const itemLabel: (target: {
56
56
  readonly label: Label;
57
57
  readonly lines?: ReadonlyArray<Range>;
58
58
  }, item: Item) => Label;
59
+ /**
60
+ * Labels every item emitted for one target together, preventing nested-range artifacts: when a violating
61
+ * target emits both an enclosing unit and a smaller item strictly nested inside it (e.g. a clean helper
62
+ * closure inside a 100-line violating function), the nested item stays `complies` if every gold range it
63
+ * overlaps is covered < 50% by it and strictly better covered by an enclosing emitted item.
64
+ */
65
+ export declare const labelFileItems: (target: {
66
+ readonly label: Label;
67
+ readonly lines?: ReadonlyArray<Range>;
68
+ }, items: ReadonlyArray<Item>) => Array<Label>;
59
69
  /** How many of `ranges` some item covers; items without lines cover every range. */
60
70
  export declare const reachedRanges: (ranges: ReadonlyArray<Range>, items: ReadonlyArray<Item>) => number;
61
71
  /**
@@ -92,6 +102,18 @@ export type ScoreVerdict = "pass" | "review" | "violation";
92
102
  export declare const verdict: (score: number, { floor, threshold }: Thresholds) => ScoreVerdict;
93
103
  /** Separation of clear violations from clean units (ambiguous units excluded). */
94
104
  export declare const auroc: (cases: ReadonlyArray<Scored<unknown>>) => number;
105
+ /**
106
+ * Area under the precision-recall curve (Average Precision) on clear cases (ambiguous excluded). Ties are
107
+ * handled by grouping equal scores so ordering within ties cannot inflate or deflate precision.
108
+ */
109
+ export declare const auprc: (cases: ReadonlyArray<Scored<unknown>>) => number;
110
+ /**
111
+ * True positive rate at a target false-positive rate `fpr` on clear cases: share of violations scored
112
+ * strictly above the `(1 - fpr)` clean quantile (with linear handling of boundary ties).
113
+ */
114
+ export declare const tprAtFpr: (cases: ReadonlyArray<Scored<unknown>>, fpr: number) => number;
115
+ /** Precision of `violation` findings (`score >= threshold`) on clear cases (ambiguous excluded). */
116
+ export declare const violationPrecision: (cases: ReadonlyArray<Scored<unknown>>, thresholds?: Thresholds) => number;
95
117
  /**
96
118
  * Ordinal concordance: over every pair of units with different targets (clean < ambiguous < violates),
97
119
  * the share where the higher target gets the higher score. Rewards placing ambiguous units between.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@andrueandersoncs/lint-anything",
3
- "version": "0.7.0",
3
+ "version": "0.9.0",
4
4
  "description": "Check anything against plain-English policies, judged by the decision model of your choice: any System One, OpenAI Decisions, or Gemini Decisions endpoint, or Workers AI",
5
5
  "license": "MIT",
6
6
  "author": "Andrue Anderson",
@@ -33,7 +33,7 @@
33
33
  "access": "public"
34
34
  },
35
35
  "scripts": {
36
- "build": "bun build index.ts --target=bun --minify --outfile dist/cli.js && bun build score.ts --target=bun --minify --outfile skills/lint-anything/scripts/score.js && bun build src/lib.ts --target=bun --external effect --external 'effect/*' --outfile dist/lib.js && rm -rf dist/types && tsc -p tsconfig.lib.json",
36
+ "build": "bun build index.ts --target=bun --minify --outfile dist/cli.js && bun build score.ts --target=bun --minify --outfile skills/lint-anything/scripts/score.js && bun build select-steps.ts --target=bun --minify --outfile skills/lint-anything/scripts/select.js && bun build src/lib.ts --target=bun --external effect --external 'effect/*' --outfile dist/lib.js && rm -rf dist/types && tsc -p tsconfig.lib.json",
37
37
  "prepublishOnly": "bunx tsc --noEmit && bun run build"
38
38
  },
39
39
  "peerDependencies": {
@@ -24,6 +24,7 @@ If `lint-anything` isn't found, install it with `bun add -d @andrueandersoncs/li
24
24
  - **The working directory matters.** Policies are read from `./.lint-anything/rules`, and selectors run in the current directory. Run from a subdirectory and you get `error: cannot read policies in .lint-anything/rules`: there is no upward search.
25
25
  - **The decision model** is the project's choice, set in the environment or in `.env` / `.env.local` in the current directory. Don't change it to make findings go away. With `LINT_ANYTHING_ENDPOINT` (a full URL), the model named by `LINT_ANYTHING_MODEL` is asked there, and `LINT_ANYTHING_API_KEY` is its key if it needs one. An endpoint on `generativelanguage.googleapis.com` speaks Google's Decisions API (`gemini-decision-1`, keyed by a Gemini API key); any other whose path ends in `/decisions` speaks OpenAI's; any other speaks System One (`/v1/systemone`: Jev, LiteLLM, LLM Gateway, and most decision-model servers).
26
26
  - **Without an endpoint**, a Cloudflare model on Workers AI is asked: `clef`, or another that `LINT_ANYTHING_MODEL` names (`clef-flash`). `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_AUTH_TOKEN` are then required on every run, even when every answer would come from the cache. `CLOUDFLARE_API_BASE_URL` is optional.
27
+ - **Optional cross-family cascading**: setting `LINT_ANYTHING_SECONDARY_MODEL` (plus optionally `LINT_ANYTHING_SECONDARY_ENDPOINT` and `LINT_ANYTHING_SECONDARY_API_KEY`; or `LINT_ANYTHING_SECONDARY_MODEL=clef` alone for Workers AI) asks the primary model first and escalates only uncertain items in `[0.10, 0.90]` to the secondary model, averaging both answers. On the 216-file unseen holdout (H32/H34), `Gemini Decision 1 → Clef` cut false alarms nearly in half (`12.8% → 6.8%`) and cost less than single-model Clef while sharing the same SQLite cache.
27
28
  - **What the model can see** besides text depends on it: Gemini Decision 1 and `clef-omni` take images, audio, and video; OpenAI Decisions, `clef`, and `clef-flash` take images only; another System One server takes what it takes. An item whose media the model can't take stops the run before anything is sent (`Clef takes text and images, not audio clips: …`).
28
29
  - **Accuracy numbers in this skill were measured on Clef**, on text. A project on another model, or with media, should score its policies on its labels before trusting them.
29
30
  - **Add `.lint-anything/cache` to `.gitignore`.** The answer cache (`answers.sqlite` plus `-wal`/`-shm` files) is written there.
@@ -124,8 +125,8 @@ Frontmatter keys are `select` and `criteria` (with both `true` and `false`), bot
124
125
 
125
126
  The model's "yes" is read as **"this item violates the rule"**:
126
127
 
127
- - `criteria.true` describes an item that **violates** the rule.
128
- - `criteria.false` describes an item that **follows** it, **or that the rule doesn't govern**. Selectors usually over-select, so say what the false answer covers.
128
+ - `criteria.true` describes an item that **violates** the rule — be rule-specific rather than generic ("The item contains a violation"). On the fresh 216-file holdout (H28/H34), replacing generic criteria with rule-specific `criteria.true` and exemption-explicit `criteria.false` plus targeted selector fixes raised Applicable File AUROC `+0.015`, Localized Caught `+6.5 pts`, and TPR@5% `+6.5 pts` outside noise at the same false-alarm rate.
129
+ - `criteria.false` describes an item that **follows** it, **its explicit exemptions**, **or items the rule doesn't govern**. Selectors usually over-select, and decision models read `criteria` immediately adjacent to every question.
129
130
 
130
131
  If you swap them, every verdict inverts. Nothing catches this but the evidence: before relying on a new policy, run it on at least one known violation and one known-clean item and check that the scores point the right way.
131
132
 
@@ -198,7 +199,19 @@ rg --json -U -P '^(?:[ \t]*(?://|#).*\n)+.*' -g '*.{ts,py,go}' |
198
199
  lines: [.data.line_number, .data.line_number + ($t | split("\n") | length) - 1], text: $t}'
199
200
  ```
200
201
 
201
- Structural units with ast-grep:
202
+ Structural units with the bundled selector helper (`scripts/select.js`, run `--help` to see every step and flag):
203
+
204
+ ```yaml
205
+ select: |
206
+ rg --files -g '*.{ts,tsx,js,py,go}' |
207
+ bun <this skill's directory>/scripts/select.js functions --flags --no-call-items |
208
+ bun <this skill's directory>/scripts/select.js file-role |
209
+ bun <this skill's directory>/scripts/select.js present --no-lines
210
+ ```
211
+
212
+ `scripts/select.js` bundles the eval-proven file cutters (`functions`, `comments`, `errors`, `errors-go`, `literals`, `module-exports`, `module-state`, `module-outline`, `tests`, `duplicates`, `excerpts`, `whole-file`) and pipe steps (`file-role`, `enclosing`, `blame`, `present`). Steps that parse ASTs resolve `ast-grep` from `PATH` or `./node_modules/.bin/ast-grep` (install with `bun add -d @ast-grep/cli`). Copy `scripts/select.js` into `.lint-anything/lib/select.js` (or reference it from `node_modules/@andrueandersoncs/lint-anything/skills/lint-anything/scripts/select.js`) so CI and teammates share the exact same selector path. Pass `--no-lines` to `present` to omit `(lines start-end)` from `text` and preserve cache hits when lines above shift.
213
+
214
+ Or write a custom ast-grep selector directly:
202
215
 
203
216
  ```sh
204
217
  ast-grep run -p 'function $NAME($$$PARAMS) { $$$BODY }' -l ts --json=stream |
@@ -265,7 +278,7 @@ One JSON object per line, one per labeled file:
265
278
  - Where they disagree, or any of them calls it ambiguous, **ask the user** about that file, showing the rule, the code, and each model's reasoning. Record what the user decides; if the user says reasonable reviewers could disagree, record `ambiguous`. Ambiguous files are left out of AUROC and counted as right when they land in `review`.
266
279
  - One model alone labels with its own reading of the rule, and tuning against those labels only makes the policy agree with that reading.
267
280
 
268
- The lint-anything repository has worked examples: 21 policies in `evals/corpus/policies/` and the scripts their selectors call in `evals/steps/`. Its evals score them the same way, against a labeled corpus. `$EVAL_STEPS` exists only inside those evals; copy a script into your project before using it.
281
+ The lint-anything repository has worked examples: 21 policies in `evals/corpus/policies/` and the source of the bundled `scripts/select.js` helpers in `evals/steps/`. Its evals score them the same way, against a labeled corpus. `$EVAL_STEPS` exists only inside those evals; in real projects use `scripts/select.js` (or copy it into `.lint-anything/lib/`).
269
282
 
270
283
  ## Cache and cost
271
284