@andrueandersoncs/lint-anything 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +65 -65
- package/dist/lib.js +212 -13
- package/dist/types/decision-model.d.ts +15 -0
- package/dist/types/lib.d.ts +2 -2
- package/dist/types/score.d.ts +22 -0
- package/package.json +1 -1
- package/skills/lint-anything/SKILL.md +2 -2
- package/skills/lint-anything/scripts/score.js +43 -43
package/dist/lib.js
CHANGED
|
@@ -29,8 +29,13 @@ var maxImageBytes = 4 * 1024 * 1024;
|
|
|
29
29
|
var maxMediaBytes = 8 * 1024 * 1024;
|
|
30
30
|
var readMedium = async (path) => {
|
|
31
31
|
const { type, kind } = mediaTypeOf(path);
|
|
32
|
-
const
|
|
33
|
-
|
|
32
|
+
const hasher = new Bun.CryptoHasher("sha256");
|
|
33
|
+
let bytes = 0;
|
|
34
|
+
for await (const chunk of Bun.file(path).stream()) {
|
|
35
|
+
hasher.update(chunk);
|
|
36
|
+
bytes += chunk.byteLength;
|
|
37
|
+
}
|
|
38
|
+
return { path, type, kind, bytes, digest: hasher.digest("hex") };
|
|
34
39
|
};
|
|
35
40
|
var mediaBase64 = async (medium) => Buffer.from(await Bun.file(medium.path).bytes()).toString("base64");
|
|
36
41
|
|
|
@@ -353,15 +358,74 @@ var failureMessage = (name, error) => {
|
|
|
353
358
|
const detail = (error instanceof Error ? error.message : String(error)).replace(/\s*\n\s*/g, " ");
|
|
354
359
|
return Effect.succeed(`${name} sent an unexpected response: ${detail}`);
|
|
355
360
|
};
|
|
361
|
+
var jsonRetryDelayMs = (text) => {
|
|
362
|
+
const match = /"retryDelay":\s*"(\d+(?:\.\d+)?)s"/.exec(text);
|
|
363
|
+
return match === null ? undefined : Math.ceil(Number(match[1]) * 1000);
|
|
364
|
+
};
|
|
356
365
|
|
|
357
366
|
class DecisionModel extends Context.Service()("lint-anything/DecisionModel") {
|
|
358
367
|
static layer = (cacheDirectory) => Layer.effect(DecisionModel, Effect.flatMap(configuredTarget, (target) => make(target, cacheDirectory))).pipe(Layer.provide(FetchHttpClient.layer));
|
|
359
368
|
static layerFor = (target, cacheDirectory) => Layer.effect(DecisionModel, make(target, cacheDirectory)).pipe(Layer.provide(FetchHttpClient.layer));
|
|
369
|
+
static cascadeLayer = (primary, secondary, cacheDirectory, options) => Layer.effect(DecisionModel, makeCascade(primary, secondary, cacheDirectory, options)).pipe(Layer.provide(FetchHttpClient.layer));
|
|
360
370
|
}
|
|
371
|
+
var makeCascade = Effect.fnUntraced(function* (primaryTarget, secondaryTarget, cacheDirectory, options) {
|
|
372
|
+
const low = options?.low ?? 0.15;
|
|
373
|
+
const high = options?.high ?? 0.85;
|
|
374
|
+
const combine = options?.combine ?? ((a, b) => (a + b) / 2);
|
|
375
|
+
const primary = yield* make(primaryTarget, cacheDirectory);
|
|
376
|
+
const secondary = yield* make(secondaryTarget, cacheDirectory);
|
|
377
|
+
return DecisionModel.of({
|
|
378
|
+
target: {
|
|
379
|
+
...primary.target,
|
|
380
|
+
name: `${primary.target.name}+${secondary.target.name}`,
|
|
381
|
+
cacheKey: `${primary.target.cacheKey}+${secondary.target.cacheKey}`,
|
|
382
|
+
inputTokenPrice: undefined
|
|
383
|
+
},
|
|
384
|
+
ask: Effect.fn("DecisionModel.cascade.ask")(function* (source, state, questions) {
|
|
385
|
+
const first = [...yield* primary.ask(source, state, questions)];
|
|
386
|
+
const escalate = [];
|
|
387
|
+
first.forEach((result, index) => {
|
|
388
|
+
if (Result.isSuccess(result) && result.success >= low && result.success <= high) {
|
|
389
|
+
escalate.push(index);
|
|
390
|
+
}
|
|
391
|
+
});
|
|
392
|
+
if (escalate.length === 0)
|
|
393
|
+
return first;
|
|
394
|
+
const second = yield* secondary.ask(source, state, escalate.map((index) => questions[index]));
|
|
395
|
+
escalate.forEach((origIndex, subIndex) => {
|
|
396
|
+
const a = first[origIndex];
|
|
397
|
+
const b = second[subIndex];
|
|
398
|
+
if (Result.isSuccess(a) && Result.isSuccess(b)) {
|
|
399
|
+
first[origIndex] = Result.succeed(combine(a.success, b.success));
|
|
400
|
+
}
|
|
401
|
+
});
|
|
402
|
+
return first;
|
|
403
|
+
}),
|
|
404
|
+
stats: Effect.map(Effect.all([primary.stats, secondary.stats]), ([a, b]) => {
|
|
405
|
+
const merged = new Map(a);
|
|
406
|
+
for (const [source, usage] of b) {
|
|
407
|
+
merged.set(source, sumUsage([merged.get(source) ?? noUsage, usage]));
|
|
408
|
+
}
|
|
409
|
+
return merged;
|
|
410
|
+
})
|
|
411
|
+
});
|
|
412
|
+
});
|
|
361
413
|
var make = Effect.fnUntraced(function* (target, cacheDirectory) {
|
|
362
414
|
const wire = wires[target.format];
|
|
363
415
|
const { token } = target;
|
|
364
|
-
const client = (yield* HttpClient.HttpClient).pipe(HttpClient.mapRequest(flow(token === undefined ? (request) => request : HttpClientRequest.setHeader(...wire.keyHeader(Redacted.value(token))), HttpClientRequest.acceptJson)), HttpClient.filterStatusOk, HttpClient.transformResponse(flow(Effect.tap((response) => response.text), Effect.timeout(attemptTimeout)
|
|
416
|
+
const client = (yield* HttpClient.HttpClient).pipe(HttpClient.mapRequest(flow(token === undefined ? (request) => request : HttpClientRequest.setHeader(...wire.keyHeader(Redacted.value(token))), HttpClientRequest.acceptJson)), HttpClient.filterStatusOk, HttpClient.transformResponse(flow(Effect.tap((response) => response.text), Effect.timeout(attemptTimeout), Effect.tapError((error) => {
|
|
417
|
+
if (!HttpClientError.isHttpClientError(error) || error.reason._tag !== "StatusCodeError")
|
|
418
|
+
return Effect.void;
|
|
419
|
+
const { response } = error.reason;
|
|
420
|
+
if (response.status !== 429 || response.headers["retry-after"] !== undefined)
|
|
421
|
+
return Effect.void;
|
|
422
|
+
return response.text.pipe(Effect.timeout("5 seconds"), Effect.flatMap((text) => {
|
|
423
|
+
const delayMs = jsonRetryDelayMs(text);
|
|
424
|
+
return delayMs !== undefined && delayMs > 0 ? Effect.sleep(delayMs) : Effect.void;
|
|
425
|
+
}), Effect.orElseSucceed(() => {
|
|
426
|
+
return;
|
|
427
|
+
}));
|
|
428
|
+
}))), HttpClient.retryTransient({ schedule: Schedule.exponential(500), times: attemptRetries }));
|
|
365
429
|
const cache = cacheDirectory === undefined ? undefined : yield* Effect.acquireRelease(Effect.try({
|
|
366
430
|
try: () => openAnswerCache(cacheDirectory, Date.now(), target),
|
|
367
431
|
catch: (error) => new DecisionModelError({
|
|
@@ -453,21 +517,71 @@ var make = Effect.fnUntraced(function* (target, cacheDirectory) {
|
|
|
453
517
|
else
|
|
454
518
|
results[index] = Result.fail(tooMuch);
|
|
455
519
|
}
|
|
456
|
-
const
|
|
457
|
-
|
|
458
|
-
|
|
520
|
+
const uniqueSendable = [];
|
|
521
|
+
const duplicates = new Map;
|
|
522
|
+
const byFingerprint = new Map;
|
|
523
|
+
for (const index of sendable) {
|
|
524
|
+
const fingerprint = `${JSON.stringify(noul(questions[index]))}\x00${prints[index]}`;
|
|
525
|
+
const leader = byFingerprint.get(fingerprint);
|
|
526
|
+
if (leader === undefined) {
|
|
527
|
+
const slot = uniqueSendable.length;
|
|
528
|
+
byFingerprint.set(fingerprint, slot);
|
|
529
|
+
uniqueSendable.push(index);
|
|
530
|
+
duplicates.set(slot, [index]);
|
|
531
|
+
} else {
|
|
532
|
+
duplicates.get(leader).push(index);
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
const minWarmBatch = 5;
|
|
536
|
+
const paddingIndices = [];
|
|
537
|
+
if (target.format === "system-one" && uniqueSendable.length > 0 && uniqueSendable.length < minWarmBatch && uniqueSendable.every((index) => (questions[index].media?.length ?? 0) === 0)) {
|
|
538
|
+
const siblingPool = [];
|
|
539
|
+
for (let i = 0;i < questions.length; i++) {
|
|
540
|
+
if (cached[i] !== undefined && (questions[i].media?.length ?? 0) === 0 && questions[i].instructions.length <= 4000) {
|
|
541
|
+
siblingPool.push(i);
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
const needed = minWarmBatch - uniqueSendable.length;
|
|
545
|
+
if (siblingPool.length > 0) {
|
|
546
|
+
const step = Math.max(1, Math.floor(siblingPool.length / needed));
|
|
547
|
+
for (let i = 0;i < siblingPool.length && paddingIndices.length < needed; i += step) {
|
|
548
|
+
paddingIndices.push(siblingPool[i]);
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
let paddingOffset = paddingIndices.length;
|
|
553
|
+
let planned = planRequests(state, [...paddingIndices.map((index) => questions[index]), ...uniqueSendable.map((index) => questions[index])], target);
|
|
554
|
+
if (paddingOffset > 0 && planned.requests.length > 1) {
|
|
555
|
+
paddingOffset = 0;
|
|
556
|
+
planned = planRequests(state, uniqueSendable.map((index) => questions[index]), target);
|
|
557
|
+
}
|
|
558
|
+
const { requests, oversized } = planned;
|
|
559
|
+
for (const { value, bytes } of oversized) {
|
|
560
|
+
if (value < paddingOffset)
|
|
561
|
+
continue;
|
|
562
|
+
const failure = Result.fail(tooLarge(bytes));
|
|
563
|
+
for (const index of duplicates.get(value - paddingOffset))
|
|
564
|
+
results[index] = failure;
|
|
565
|
+
}
|
|
459
566
|
yield* Effect.forEach(requests, (request) => send(source, state, request, media).pipe(Effect.tap((received) => {
|
|
460
567
|
const answered = [];
|
|
461
568
|
for (const { value, answer } of received) {
|
|
462
|
-
|
|
569
|
+
if (value < paddingOffset)
|
|
570
|
+
continue;
|
|
571
|
+
const targets = duplicates.get(value - paddingOffset);
|
|
572
|
+
const leader = targets[0];
|
|
463
573
|
if (answer === "declined") {
|
|
464
|
-
|
|
574
|
+
const failure = Result.fail(declined);
|
|
575
|
+
for (const index of targets)
|
|
576
|
+
results[index] = failure;
|
|
465
577
|
} else {
|
|
466
|
-
|
|
467
|
-
|
|
578
|
+
const success = Result.succeed(answer);
|
|
579
|
+
for (const index of targets)
|
|
580
|
+
results[index] = success;
|
|
581
|
+
answered.push({ question: questions[leader], print: prints[leader], noul: answer });
|
|
468
582
|
}
|
|
469
583
|
}
|
|
470
|
-
return cache === undefined ? Effect.void : cache.set(state, answered);
|
|
584
|
+
return cache === undefined || answered.length === 0 ? Effect.void : cache.set(state, answered);
|
|
471
585
|
})), { concurrency: "unbounded", discard: true });
|
|
472
586
|
return results;
|
|
473
587
|
});
|
|
@@ -619,7 +733,8 @@ var itemLocation = (item) => ({
|
|
|
619
733
|
...item.path === undefined ? {} : { path: item.path },
|
|
620
734
|
...item.lines === undefined ? {} : { lines: item.lines }
|
|
621
735
|
});
|
|
622
|
-
var
|
|
736
|
+
var compareStrings = (a, b) => a < b ? -1 : a > b ? 1 : 0;
|
|
737
|
+
var byLocation = (a, b) => compareStrings(a.path ?? "", b.path ?? "") || (a.lines?.[0] ?? 0) - (b.lines?.[0] ?? 0) || compareStrings(a.id, b.id);
|
|
623
738
|
var select = Effect4.fn("select")(function* (policy, cwd, runs) {
|
|
624
739
|
return (yield* runSelector(policy, cwd, runs)).toSorted(byLocation);
|
|
625
740
|
});
|
|
@@ -754,7 +869,35 @@ var loadLabels = Effect6.fn("loadLabels")(function* (policyName, directory = ".l
|
|
|
754
869
|
return yield* parseLabels(file, text);
|
|
755
870
|
});
|
|
756
871
|
var overlaps = (lines, [start, end]) => lines[0] <= end && start <= lines[1];
|
|
872
|
+
var overlapLength = ([a0, a1], [b0, b1]) => Math.max(0, Math.min(a1, b1) - Math.max(a0, b0) + 1);
|
|
757
873
|
var itemLabel = (target, item) => target.label !== "violates" || target.lines === undefined ? target.label : target.lines.some((range) => item.lines === undefined || overlaps(item.lines, range)) ? "violates" : "complies";
|
|
874
|
+
var labelFileItems = (target, items) => {
|
|
875
|
+
if (target.label !== "violates" || target.lines === undefined)
|
|
876
|
+
return items.map(() => target.label);
|
|
877
|
+
const ranges = target.lines;
|
|
878
|
+
return items.map((item) => {
|
|
879
|
+
if (item.lines === undefined)
|
|
880
|
+
return "violates";
|
|
881
|
+
const [start, end] = item.lines;
|
|
882
|
+
const matched = ranges.filter((range) => overlaps(item.lines, range));
|
|
883
|
+
if (matched.length === 0)
|
|
884
|
+
return "complies";
|
|
885
|
+
const onlyNested = matched.every((range) => {
|
|
886
|
+
const rangeLen = range[1] - range[0] + 1;
|
|
887
|
+
const mine = overlapLength(item.lines, range);
|
|
888
|
+
if (mine / rangeLen >= 0.5)
|
|
889
|
+
return false;
|
|
890
|
+
return items.some((other) => {
|
|
891
|
+
if (other === item || other.lines === undefined)
|
|
892
|
+
return false;
|
|
893
|
+
const [os, oe] = other.lines;
|
|
894
|
+
const strictlyEncloses = os <= start && end <= oe && (os < start || oe > end);
|
|
895
|
+
return strictlyEncloses && overlapLength(other.lines, range) > mine;
|
|
896
|
+
});
|
|
897
|
+
});
|
|
898
|
+
return onlyNested ? "complies" : "violates";
|
|
899
|
+
});
|
|
900
|
+
};
|
|
758
901
|
var reachedRanges = (ranges, items) => ranges.filter((range) => items.some((item) => item.lines === undefined || overlaps(item.lines, range))).length;
|
|
759
902
|
var targetScore = (scores) => {
|
|
760
903
|
const answered = scores.filter((score) => score !== null);
|
|
@@ -784,6 +927,55 @@ var separation = (low, high) => {
|
|
|
784
927
|
};
|
|
785
928
|
var scoresOf = (cases, label) => cases.filter((scored) => scored.label === label).map(probability);
|
|
786
929
|
var auroc = (cases) => separation(scoresOf(cases, "complies"), scoresOf(cases, "violates"));
|
|
930
|
+
var auprc = (cases) => {
|
|
931
|
+
const clear = cases.filter((scored) => scored.label !== "ambiguous");
|
|
932
|
+
const totalPos = clear.filter((scored) => scored.label === "violates").length;
|
|
933
|
+
if (totalPos === 0 || clear.length === totalPos)
|
|
934
|
+
return Number.NaN;
|
|
935
|
+
const sorted = clear.map((scored) => ({ score: probability(scored), pos: scored.label === "violates" })).sort((a, b) => b.score - a.score);
|
|
936
|
+
let tp = 0;
|
|
937
|
+
let seen = 0;
|
|
938
|
+
let ap = 0;
|
|
939
|
+
for (let start = 0;start < sorted.length; ) {
|
|
940
|
+
let end = start;
|
|
941
|
+
let groupPos = 0;
|
|
942
|
+
while (end < sorted.length && sorted[end].score === sorted[start].score) {
|
|
943
|
+
if (sorted[end].pos)
|
|
944
|
+
groupPos++;
|
|
945
|
+
end++;
|
|
946
|
+
}
|
|
947
|
+
const groupSize = end - start;
|
|
948
|
+
if (groupPos > 0) {
|
|
949
|
+
let sumPrec = 0;
|
|
950
|
+
for (let k = 1;k <= groupPos; k++) {
|
|
951
|
+
const expectedSeen = seen + k * (groupSize + 1) / (groupPos + 1);
|
|
952
|
+
sumPrec += (tp + k) / expectedSeen;
|
|
953
|
+
}
|
|
954
|
+
ap += sumPrec / totalPos;
|
|
955
|
+
}
|
|
956
|
+
tp += groupPos;
|
|
957
|
+
seen += groupSize;
|
|
958
|
+
start = end;
|
|
959
|
+
}
|
|
960
|
+
return ap;
|
|
961
|
+
};
|
|
962
|
+
var tprAtFpr = (cases, fpr) => {
|
|
963
|
+
const clean = scoresOf(cases, "complies").sort((a, b) => b - a);
|
|
964
|
+
const viol = scoresOf(cases, "violates");
|
|
965
|
+
if (clean.length === 0 || viol.length === 0)
|
|
966
|
+
return Number.NaN;
|
|
967
|
+
const cut = clean[Math.min(clean.length - 1, Math.floor(clean.length * fpr))];
|
|
968
|
+
const above = viol.filter((value) => value > cut).length;
|
|
969
|
+
const tied = viol.filter((value) => value === cut).length;
|
|
970
|
+
const cleanAbove = clean.filter((value) => value > cut).length;
|
|
971
|
+
const cleanTied = clean.filter((value) => value === cut).length;
|
|
972
|
+
const allowedFromTie = cleanTied > 0 ? Math.max(0, Math.min(1, (fpr * clean.length - cleanAbove) / cleanTied)) : 0;
|
|
973
|
+
return (above + tied * allowedFromTie) / viol.length;
|
|
974
|
+
};
|
|
975
|
+
var violationPrecision = (cases, thresholds = cliThresholds) => {
|
|
976
|
+
const clearFlagged = cases.filter((scored) => scored.label !== "ambiguous" && verdict(probability(scored), thresholds) === "violation");
|
|
977
|
+
return clearFlagged.length === 0 ? Number.NaN : clearFlagged.filter((scored) => scored.label === "violates").length / clearFlagged.length;
|
|
978
|
+
};
|
|
787
979
|
var concordance = (cases) => {
|
|
788
980
|
const clean = scoresOf(cases, "complies");
|
|
789
981
|
const ambiguous = scoresOf(cases, "ambiguous");
|
|
@@ -918,7 +1110,10 @@ var scorePolicy = Effect6.fn("scorePolicy")(function* (policy, labels, cwd) {
|
|
|
918
1110
|
}
|
|
919
1111
|
return {
|
|
920
1112
|
targets: ofTarget.map(({ record, items }) => ({ subject: record, label: record.label, group: targetKey(record), score: targetScore(items.map(score)) })),
|
|
921
|
-
items: ofTarget.flatMap(({ record, items }) =>
|
|
1113
|
+
items: ofTarget.flatMap(({ record, items }) => {
|
|
1114
|
+
const itemLabels = labelFileItems(record, items);
|
|
1115
|
+
return items.map((item, index) => ({ subject: item, label: itemLabels[index], group: targetKey(record), score: score(item) }));
|
|
1116
|
+
}),
|
|
922
1117
|
reach
|
|
923
1118
|
};
|
|
924
1119
|
});
|
|
@@ -936,6 +1131,7 @@ export {
|
|
|
936
1131
|
PolicyError,
|
|
937
1132
|
RequestTooLarge,
|
|
938
1133
|
SelectorError,
|
|
1134
|
+
auprc,
|
|
939
1135
|
auroc,
|
|
940
1136
|
bootstrapInterval,
|
|
941
1137
|
brier,
|
|
@@ -959,6 +1155,7 @@ export {
|
|
|
959
1155
|
judge,
|
|
960
1156
|
keyHeader,
|
|
961
1157
|
killRunningSelectors,
|
|
1158
|
+
labelFileItems,
|
|
962
1159
|
lint,
|
|
963
1160
|
loadLabels,
|
|
964
1161
|
loadPolicies,
|
|
@@ -978,9 +1175,11 @@ export {
|
|
|
978
1175
|
sumUsage,
|
|
979
1176
|
targetKey,
|
|
980
1177
|
targetScore,
|
|
1178
|
+
tprAtFpr,
|
|
981
1179
|
tuneThresholds,
|
|
982
1180
|
verdict,
|
|
983
1181
|
verdictF1,
|
|
984
1182
|
verdictOf,
|
|
1183
|
+
violationPrecision,
|
|
985
1184
|
workersAiTarget
|
|
986
1185
|
};
|
|
@@ -177,6 +177,15 @@ export interface Usage {
|
|
|
177
177
|
}
|
|
178
178
|
export declare const noUsage: Usage;
|
|
179
179
|
export declare const sumUsage: (usages: Iterable<Usage>) => Usage;
|
|
180
|
+
/** Configuration for cascading two decision models: a fast primary and a secondary tiebreaker. */
|
|
181
|
+
export interface CascadeOptions {
|
|
182
|
+
/** Primary probabilities below `low` are accepted directly without calling `secondary` (default: 0.15). */
|
|
183
|
+
readonly low?: number;
|
|
184
|
+
/** Primary probabilities above `high` are accepted directly without calling `secondary` (default: 0.85). */
|
|
185
|
+
readonly high?: number;
|
|
186
|
+
/** Combines primary and secondary probabilities for items sent to both (default: arithmetic mean). */
|
|
187
|
+
readonly combine?: (primary: number, secondary: number) => number;
|
|
188
|
+
}
|
|
180
189
|
declare const DecisionModel_base: Context.ServiceClass<DecisionModel, "lint-anything/DecisionModel", {
|
|
181
190
|
/** The model asked, and where. */
|
|
182
191
|
readonly target: Target;
|
|
@@ -197,5 +206,11 @@ export declare class DecisionModel extends DecisionModel_base {
|
|
|
197
206
|
static readonly layer: (cacheDirectory?: string) => Layer.Layer<DecisionModel, DecisionModelError, never>;
|
|
198
207
|
/** `target`, whatever the environment says; answers are cached as with `layer`. */
|
|
199
208
|
static readonly layerFor: (target: Target, cacheDirectory?: string) => Layer.Layer<DecisionModel, DecisionModelError, never>;
|
|
209
|
+
/**
|
|
210
|
+
* Cross-family cascade: asks `primary` first, and asks `secondary` only for questions whose primary
|
|
211
|
+
* probability lands in `[low, high]`, combining both probabilities with `combine` (default: mean).
|
|
212
|
+
* Combines `stats` from both models.
|
|
213
|
+
*/
|
|
214
|
+
static readonly cascadeLayer: (primary: Target, secondary: Target, cacheDirectory?: string, options?: CascadeOptions) => Layer.Layer<DecisionModel, DecisionModelError, never>;
|
|
200
215
|
}
|
|
201
216
|
export {};
|
package/dist/types/lib.d.ts
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* with the metrics and bootstrap intervals the evals use; the skill's score script is built on it.
|
|
19
19
|
*/
|
|
20
20
|
import { DecisionModel, DecisionModelError } from "./decision-model.ts";
|
|
21
|
-
export { cacheMaxAgeMs, clef, clefInputTokenPrice, configuredModel, configuredTarget, Declined, declinesAll, decodeReply, DecisionModel, DecisionModelError, endpointTarget, type Format, keyHeader, type Model, noUsage, planRequests, type Planned, type Question, RequestTooLarge, sumUsage, type Target, type Unanswered, type Usage, workersAiTarget } from "./decision-model.ts";
|
|
21
|
+
export { cacheMaxAgeMs, type CascadeOptions, clef, clefInputTokenPrice, configuredModel, configuredTarget, Declined, declinesAll, decodeReply, DecisionModel, DecisionModelError, endpointTarget, type Format, keyHeader, type Model, noUsage, planRequests, type Planned, type Question, RequestTooLarge, sumUsage, type Target, type Unanswered, type Usage, workersAiTarget } from "./decision-model.ts";
|
|
22
22
|
/** @deprecated Renamed `DecisionModel`: it asks whichever decision model is configured, Clef by default. */
|
|
23
23
|
export declare const Clef: typeof DecisionModel;
|
|
24
24
|
/** @deprecated Renamed `DecisionModel`. */
|
|
@@ -31,4 +31,4 @@ export { type Item, parseItem } from "./item.ts";
|
|
|
31
31
|
export { decide, defaultThreshold, type Finding, itemQuestion, judge, lint, reviewFloor, select, type Verdict, verdictOf } from "./lint.ts";
|
|
32
32
|
export { loadPolicies, parsePolicy, type Policy, PolicyError } from "./policy.ts";
|
|
33
33
|
export { killRunningSelectors, runSelector, SelectorError, type SelectorRuns, selectorTimeoutMinutes } from "./select.ts";
|
|
34
|
-
export { auroc, bootstrapInterval, brier, caught, cliThresholds, concordance, falseAlarms, itemLabel, itemsOf, type Label, LabelError, type LabelRecord, loadLabels, pairedInterval, parseLabels, type PolicyScore, type Quality, quality, type Range, reachedRanges, type Scored, scorePolicy, type ScoreVerdict, type Split, targetKey, targetScore, type Thresholds, tuneThresholds, verdict, verdictF1 } from "./score.ts";
|
|
34
|
+
export { auprc, auroc, bootstrapInterval, brier, caught, cliThresholds, concordance, falseAlarms, itemLabel, itemsOf, type Label, LabelError, labelFileItems, type LabelRecord, loadLabels, pairedInterval, parseLabels, type PolicyScore, type Quality, quality, type Range, reachedRanges, type Scored, scorePolicy, type ScoreVerdict, type Split, targetKey, targetScore, type Thresholds, tprAtFpr, tuneThresholds, verdict, verdictF1, violationPrecision } from "./score.ts";
|
package/dist/types/score.d.ts
CHANGED
|
@@ -56,6 +56,16 @@ export declare const itemLabel: (target: {
|
|
|
56
56
|
readonly label: Label;
|
|
57
57
|
readonly lines?: ReadonlyArray<Range>;
|
|
58
58
|
}, item: Item) => Label;
|
|
59
|
+
/**
|
|
60
|
+
* Labels every item emitted for one target together, preventing nested-range artifacts: when a violating
|
|
61
|
+
* target emits both an enclosing unit and a smaller item strictly nested inside it (e.g. a clean helper
|
|
62
|
+
* closure inside a 100-line violating function), the nested item stays `complies` if every gold range it
|
|
63
|
+
* overlaps is covered < 50% by it and strictly better covered by an enclosing emitted item.
|
|
64
|
+
*/
|
|
65
|
+
export declare const labelFileItems: (target: {
|
|
66
|
+
readonly label: Label;
|
|
67
|
+
readonly lines?: ReadonlyArray<Range>;
|
|
68
|
+
}, items: ReadonlyArray<Item>) => Array<Label>;
|
|
59
69
|
/** How many of `ranges` some item covers; items without lines cover every range. */
|
|
60
70
|
export declare const reachedRanges: (ranges: ReadonlyArray<Range>, items: ReadonlyArray<Item>) => number;
|
|
61
71
|
/**
|
|
@@ -92,6 +102,18 @@ export type ScoreVerdict = "pass" | "review" | "violation";
|
|
|
92
102
|
export declare const verdict: (score: number, { floor, threshold }: Thresholds) => ScoreVerdict;
|
|
93
103
|
/** Separation of clear violations from clean units (ambiguous units excluded). */
|
|
94
104
|
export declare const auroc: (cases: ReadonlyArray<Scored<unknown>>) => number;
|
|
105
|
+
/**
|
|
106
|
+
* Area under the precision-recall curve (Average Precision) on clear cases (ambiguous excluded). Ties are
|
|
107
|
+
* handled by grouping equal scores so ordering within ties cannot inflate or deflate precision.
|
|
108
|
+
*/
|
|
109
|
+
export declare const auprc: (cases: ReadonlyArray<Scored<unknown>>) => number;
|
|
110
|
+
/**
|
|
111
|
+
* True positive rate at a target false-positive rate `fpr` on clear cases: share of violations scored
|
|
112
|
+
* strictly above the `(1 - fpr)` clean quantile (with linear handling of boundary ties).
|
|
113
|
+
*/
|
|
114
|
+
export declare const tprAtFpr: (cases: ReadonlyArray<Scored<unknown>>, fpr: number) => number;
|
|
115
|
+
/** Precision of `violation` findings (`score >= threshold`) on clear cases (ambiguous excluded). */
|
|
116
|
+
export declare const violationPrecision: (cases: ReadonlyArray<Scored<unknown>>, thresholds?: Thresholds) => number;
|
|
95
117
|
/**
|
|
96
118
|
* Ordinal concordance: over every pair of units with different targets (clean < ambiguous < violates),
|
|
97
119
|
* the share where the higher target gets the higher score. Rewards placing ambiguous units between.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@andrueandersoncs/lint-anything",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "Check anything against plain-English policies, judged by the decision model of your choice: any System One, OpenAI Decisions, or Gemini Decisions endpoint, or Workers AI",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Andrue Anderson",
|
|
@@ -124,8 +124,8 @@ Frontmatter keys are `select` and `criteria` (with both `true` and `false`), bot
|
|
|
124
124
|
|
|
125
125
|
The model's "yes" is read as **"this item violates the rule"**:
|
|
126
126
|
|
|
127
|
-
- `criteria.true` describes an item that **violates** the rule.
|
|
128
|
-
- `criteria.false` describes an item that **follows** it, **or
|
|
127
|
+
- `criteria.true` describes an item that **violates** the rule — be rule-specific rather than generic ("The item contains a violation"). On the fresh 216-file holdout (H28/H34), replacing generic criteria with rule-specific `criteria.true` and exemption-explicit `criteria.false` plus targeted selector fixes raised Applicable File AUROC `+0.015`, Localized Caught `+6.5 pts`, and TPR@5% `+6.5 pts` outside noise at the same false-alarm rate.
|
|
128
|
+
- `criteria.false` describes an item that **follows** it, **its explicit exemptions**, **or items the rule doesn't govern**. Selectors usually over-select, and decision models read `criteria` immediately adjacent to every question.
|
|
129
129
|
|
|
130
130
|
If you swap them, every verdict inverts. Nothing catches this but the evidence: before relying on a new policy, run it on at least one known violation and one known-clean item and check that the scores point the right way.
|
|
131
131
|
|