@andrueandersoncs/lint-anything 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -10
- package/dist/cli.js +66 -66
- package/dist/lib.js +268 -19
- package/dist/types/decision-model.d.ts +26 -4
- package/dist/types/lib.d.ts +2 -2
- package/dist/types/score.d.ts +22 -0
- package/package.json +2 -2
- package/skills/lint-anything/SKILL.md +17 -4
- package/skills/lint-anything/scripts/score.js +43 -43
- package/skills/lint-anything/scripts/select.js +351 -0
package/dist/lib.js
CHANGED
|
@@ -29,8 +29,13 @@ var maxImageBytes = 4 * 1024 * 1024;
|
|
|
29
29
|
var maxMediaBytes = 8 * 1024 * 1024;
|
|
30
30
|
var readMedium = async (path) => {
|
|
31
31
|
const { type, kind } = mediaTypeOf(path);
|
|
32
|
-
const
|
|
33
|
-
|
|
32
|
+
const hasher = new Bun.CryptoHasher("sha256");
|
|
33
|
+
let bytes = 0;
|
|
34
|
+
for await (const chunk of Bun.file(path).stream()) {
|
|
35
|
+
hasher.update(chunk);
|
|
36
|
+
bytes += chunk.byteLength;
|
|
37
|
+
}
|
|
38
|
+
return { path, type, kind, bytes, digest: hasher.digest("hex") };
|
|
34
39
|
};
|
|
35
40
|
var mediaBase64 = async (medium) => Buffer.from(await Bun.file(medium.path).bytes()).toString("base64");
|
|
36
41
|
|
|
@@ -223,16 +228,56 @@ var choice = Effect.gen(function* () {
|
|
|
223
228
|
}
|
|
224
229
|
return { kind: "workers-ai", model: name };
|
|
225
230
|
});
|
|
231
|
+
var secondaryChoice = Effect.gen(function* () {
|
|
232
|
+
const endpoint = yield* setting("LINT_ANYTHING_SECONDARY_ENDPOINT");
|
|
233
|
+
const model = yield* setting("LINT_ANYTHING_SECONDARY_MODEL");
|
|
234
|
+
const apiKey = yield* setting("LINT_ANYTHING_SECONDARY_API_KEY");
|
|
235
|
+
if (Option.isSome(endpoint)) {
|
|
236
|
+
if (!isHttpUrl(endpoint.value)) {
|
|
237
|
+
return yield* configError(`LINT_ANYTHING_SECONDARY_ENDPOINT must be an http or https URL, such as http://localhost:4000/v1/systemone`);
|
|
238
|
+
}
|
|
239
|
+
if (Option.isNone(model)) {
|
|
240
|
+
return yield* configError(`LINT_ANYTHING_SECONDARY_MODEL is not set; LINT_ANYTHING_SECONDARY_ENDPOINT needs the name of the model to ask`);
|
|
241
|
+
}
|
|
242
|
+
return Option.some({
|
|
243
|
+
kind: "endpoint",
|
|
244
|
+
url: endpoint.value,
|
|
245
|
+
model: model.value,
|
|
246
|
+
token: Option.getOrUndefined(Option.map(apiKey, Redacted.make))
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
if (Option.isSome(apiKey)) {
|
|
250
|
+
return yield* configError("LINT_ANYTHING_SECONDARY_API_KEY is set but LINT_ANYTHING_SECONDARY_ENDPOINT is not; set the endpoint the key is for (Clef on Workers AI uses CLOUDFLARE_AUTH_TOKEN)");
|
|
251
|
+
}
|
|
252
|
+
if (Option.isNone(model))
|
|
253
|
+
return Option.none();
|
|
254
|
+
const name = model.value;
|
|
255
|
+
if (!/^[\w.-]+$/.test(name)) {
|
|
256
|
+
return yield* configError(`LINT_ANYTHING_SECONDARY_MODEL ${name} is not a Cloudflare model; to ask it, set LINT_ANYTHING_SECONDARY_ENDPOINT to the decision endpoint that serves it`);
|
|
257
|
+
}
|
|
258
|
+
return Option.some({ kind: "workers-ai", model: name });
|
|
259
|
+
});
|
|
260
|
+
var resolveWorkersAi = (model) => Effect.gen(function* () {
|
|
261
|
+
const required = (name, config) => config.pipe(Effect.mapError(() => configError(`${name} is not set; set it in the environment or in .env`)));
|
|
262
|
+
const accountId = yield* required("CLOUDFLARE_ACCOUNT_ID", Config.String("CLOUDFLARE_ACCOUNT_ID"));
|
|
263
|
+
const token = yield* required("CLOUDFLARE_AUTH_TOKEN", Config.Redacted("CLOUDFLARE_AUTH_TOKEN"));
|
|
264
|
+
const baseUrl = yield* Config.String("CLOUDFLARE_API_BASE_URL").pipe(Config.withDefault(defaultCloudflareBaseUrl), Effect.mapError((error) => configError(`CLOUDFLARE_API_BASE_URL: ${error.message}`)));
|
|
265
|
+
return workersAiTarget({ accountId, token, model, baseUrl });
|
|
266
|
+
});
|
|
226
267
|
var configuredModel = Effect.map(choice, (chosen) => chosen.kind === "endpoint" ? endpointModel(chosen.url, chosen.model) : workersAiModel(chosen.model));
|
|
227
268
|
var configuredTarget = Effect.gen(function* () {
|
|
228
269
|
const chosen = yield* choice;
|
|
229
270
|
if (chosen.kind === "endpoint")
|
|
230
271
|
return endpointTarget(chosen);
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
const
|
|
235
|
-
|
|
272
|
+
return yield* resolveWorkersAi(chosen.model);
|
|
273
|
+
});
|
|
274
|
+
var configuredSecondaryTarget = Effect.gen(function* () {
|
|
275
|
+
const chosen = yield* secondaryChoice;
|
|
276
|
+
if (Option.isNone(chosen))
|
|
277
|
+
return Option.none();
|
|
278
|
+
if (chosen.value.kind === "endpoint")
|
|
279
|
+
return Option.some(endpointTarget(chosen.value));
|
|
280
|
+
return Option.some(yield* resolveWorkersAi(chosen.value.model));
|
|
236
281
|
});
|
|
237
282
|
var pack = (model, state, media, values, question) => {
|
|
238
283
|
const wire = wires[model.format];
|
|
@@ -353,15 +398,83 @@ var failureMessage = (name, error) => {
|
|
|
353
398
|
const detail = (error instanceof Error ? error.message : String(error)).replace(/\s*\n\s*/g, " ");
|
|
354
399
|
return Effect.succeed(`${name} sent an unexpected response: ${detail}`);
|
|
355
400
|
};
|
|
401
|
+
var jsonRetryDelayMs = (text) => {
|
|
402
|
+
const match = /"retryDelay":\s*"(\d+(?:\.\d+)?)s"/.exec(text);
|
|
403
|
+
return match === null ? undefined : Math.ceil(Number(match[1]) * 1000);
|
|
404
|
+
};
|
|
356
405
|
|
|
357
406
|
class DecisionModel extends Context.Service()("lint-anything/DecisionModel") {
|
|
358
|
-
static layer = (cacheDirectory) => Layer.effect(DecisionModel, Effect.
|
|
407
|
+
static layer = (cacheDirectory) => Layer.effect(DecisionModel, Effect.gen(function* () {
|
|
408
|
+
const primary = yield* configuredTarget;
|
|
409
|
+
const secondary = yield* configuredSecondaryTarget;
|
|
410
|
+
if (Option.isNone(secondary))
|
|
411
|
+
return yield* make(primary, cacheDirectory);
|
|
412
|
+
if (secondary.value.cacheKey === primary.cacheKey) {
|
|
413
|
+
return yield* configError(`secondary decision model (${secondary.value.name}) is the same as the primary model; use a different model for cascading`);
|
|
414
|
+
}
|
|
415
|
+
return yield* makeCascade(primary, secondary.value, cacheDirectory);
|
|
416
|
+
})).pipe(Layer.provide(FetchHttpClient.layer));
|
|
359
417
|
static layerFor = (target, cacheDirectory) => Layer.effect(DecisionModel, make(target, cacheDirectory)).pipe(Layer.provide(FetchHttpClient.layer));
|
|
418
|
+
static cascadeLayer = (primary, secondary, cacheDirectory, options) => Layer.effect(DecisionModel, makeCascade(primary, secondary, cacheDirectory, options)).pipe(Layer.provide(FetchHttpClient.layer));
|
|
360
419
|
}
|
|
420
|
+
var makeCascade = Effect.fnUntraced(function* (primaryTarget, secondaryTarget, cacheDirectory, options) {
|
|
421
|
+
const low = options?.low ?? 0.1;
|
|
422
|
+
const high = options?.high ?? 0.9;
|
|
423
|
+
const combine = options?.combine ?? ((a, b) => (a + b) / 2);
|
|
424
|
+
const primary = yield* make(primaryTarget, cacheDirectory);
|
|
425
|
+
const secondary = yield* make(secondaryTarget, cacheDirectory);
|
|
426
|
+
return DecisionModel.of({
|
|
427
|
+
target: {
|
|
428
|
+
...primary.target,
|
|
429
|
+
name: `${primary.target.name}+${secondary.target.name}`,
|
|
430
|
+
cacheKey: `${primary.target.cacheKey}+${secondary.target.cacheKey}`,
|
|
431
|
+
inputTokenPrice: undefined
|
|
432
|
+
},
|
|
433
|
+
ask: Effect.fn("DecisionModel.cascade.ask")(function* (source, state, questions) {
|
|
434
|
+
const first = [...yield* primary.ask(source, state, questions)];
|
|
435
|
+
const escalate = [];
|
|
436
|
+
first.forEach((result, index) => {
|
|
437
|
+
if (Result.isSuccess(result) && result.success >= low && result.success <= high) {
|
|
438
|
+
escalate.push(index);
|
|
439
|
+
}
|
|
440
|
+
});
|
|
441
|
+
if (escalate.length === 0)
|
|
442
|
+
return first;
|
|
443
|
+
const second = yield* secondary.ask(source, state, escalate.map((index) => questions[index]));
|
|
444
|
+
escalate.forEach((origIndex, subIndex) => {
|
|
445
|
+
const a = first[origIndex];
|
|
446
|
+
const b = second[subIndex];
|
|
447
|
+
if (Result.isSuccess(a) && Result.isSuccess(b)) {
|
|
448
|
+
first[origIndex] = Result.succeed(combine(a.success, b.success));
|
|
449
|
+
}
|
|
450
|
+
});
|
|
451
|
+
return first;
|
|
452
|
+
}),
|
|
453
|
+
stats: Effect.map(Effect.all([primary.stats, secondary.stats]), ([a, b]) => {
|
|
454
|
+
const merged = new Map(a);
|
|
455
|
+
for (const [source, usage] of b) {
|
|
456
|
+
merged.set(source, sumUsage([merged.get(source) ?? noUsage, usage]));
|
|
457
|
+
}
|
|
458
|
+
return merged;
|
|
459
|
+
})
|
|
460
|
+
});
|
|
461
|
+
});
|
|
361
462
|
var make = Effect.fnUntraced(function* (target, cacheDirectory) {
|
|
362
463
|
const wire = wires[target.format];
|
|
363
464
|
const { token } = target;
|
|
364
|
-
const client = (yield* HttpClient.HttpClient).pipe(HttpClient.mapRequest(flow(token === undefined ? (request) => request : HttpClientRequest.setHeader(...wire.keyHeader(Redacted.value(token))), HttpClientRequest.acceptJson)), HttpClient.filterStatusOk, HttpClient.transformResponse(flow(Effect.tap((response) => response.text), Effect.timeout(attemptTimeout)
|
|
465
|
+
const client = (yield* HttpClient.HttpClient).pipe(HttpClient.mapRequest(flow(token === undefined ? (request) => request : HttpClientRequest.setHeader(...wire.keyHeader(Redacted.value(token))), HttpClientRequest.acceptJson)), HttpClient.filterStatusOk, HttpClient.transformResponse(flow(Effect.tap((response) => response.text), Effect.timeout(attemptTimeout), Effect.tapError((error) => {
|
|
466
|
+
if (!HttpClientError.isHttpClientError(error) || error.reason._tag !== "StatusCodeError")
|
|
467
|
+
return Effect.void;
|
|
468
|
+
const { response } = error.reason;
|
|
469
|
+
if (response.status !== 429 || response.headers["retry-after"] !== undefined)
|
|
470
|
+
return Effect.void;
|
|
471
|
+
return response.text.pipe(Effect.timeout("5 seconds"), Effect.flatMap((text) => {
|
|
472
|
+
const delayMs = jsonRetryDelayMs(text);
|
|
473
|
+
return delayMs !== undefined && delayMs > 0 ? Effect.sleep(delayMs) : Effect.void;
|
|
474
|
+
}), Effect.orElseSucceed(() => {
|
|
475
|
+
return;
|
|
476
|
+
}));
|
|
477
|
+
}))), HttpClient.retryTransient({ schedule: Schedule.exponential(500), times: attemptRetries }));
|
|
365
478
|
const cache = cacheDirectory === undefined ? undefined : yield* Effect.acquireRelease(Effect.try({
|
|
366
479
|
try: () => openAnswerCache(cacheDirectory, Date.now(), target),
|
|
367
480
|
catch: (error) => new DecisionModelError({
|
|
@@ -453,21 +566,71 @@ var make = Effect.fnUntraced(function* (target, cacheDirectory) {
|
|
|
453
566
|
else
|
|
454
567
|
results[index] = Result.fail(tooMuch);
|
|
455
568
|
}
|
|
456
|
-
const
|
|
457
|
-
|
|
458
|
-
|
|
569
|
+
const uniqueSendable = [];
|
|
570
|
+
const duplicates = new Map;
|
|
571
|
+
const byFingerprint = new Map;
|
|
572
|
+
for (const index of sendable) {
|
|
573
|
+
const fingerprint = `${JSON.stringify(noul(questions[index]))}\x00${prints[index]}`;
|
|
574
|
+
const leader = byFingerprint.get(fingerprint);
|
|
575
|
+
if (leader === undefined) {
|
|
576
|
+
const slot = uniqueSendable.length;
|
|
577
|
+
byFingerprint.set(fingerprint, slot);
|
|
578
|
+
uniqueSendable.push(index);
|
|
579
|
+
duplicates.set(slot, [index]);
|
|
580
|
+
} else {
|
|
581
|
+
duplicates.get(leader).push(index);
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
const minWarmBatch = 5;
|
|
585
|
+
const paddingIndices = [];
|
|
586
|
+
if (target.format === "system-one" && uniqueSendable.length > 0 && uniqueSendable.length < minWarmBatch && uniqueSendable.every((index) => (questions[index].media?.length ?? 0) === 0)) {
|
|
587
|
+
const siblingPool = [];
|
|
588
|
+
for (let i = 0;i < questions.length; i++) {
|
|
589
|
+
if (cached[i] !== undefined && (questions[i].media?.length ?? 0) === 0 && questions[i].instructions.length <= 4000) {
|
|
590
|
+
siblingPool.push(i);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
const needed = minWarmBatch - uniqueSendable.length;
|
|
594
|
+
if (siblingPool.length > 0) {
|
|
595
|
+
const step = Math.max(1, Math.floor(siblingPool.length / needed));
|
|
596
|
+
for (let i = 0;i < siblingPool.length && paddingIndices.length < needed; i += step) {
|
|
597
|
+
paddingIndices.push(siblingPool[i]);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
let paddingOffset = paddingIndices.length;
|
|
602
|
+
let planned = planRequests(state, [...paddingIndices.map((index) => questions[index]), ...uniqueSendable.map((index) => questions[index])], target);
|
|
603
|
+
if (paddingOffset > 0 && planned.requests.length > 1) {
|
|
604
|
+
paddingOffset = 0;
|
|
605
|
+
planned = planRequests(state, uniqueSendable.map((index) => questions[index]), target);
|
|
606
|
+
}
|
|
607
|
+
const { requests, oversized } = planned;
|
|
608
|
+
for (const { value, bytes } of oversized) {
|
|
609
|
+
if (value < paddingOffset)
|
|
610
|
+
continue;
|
|
611
|
+
const failure = Result.fail(tooLarge(bytes));
|
|
612
|
+
for (const index of duplicates.get(value - paddingOffset))
|
|
613
|
+
results[index] = failure;
|
|
614
|
+
}
|
|
459
615
|
yield* Effect.forEach(requests, (request) => send(source, state, request, media).pipe(Effect.tap((received) => {
|
|
460
616
|
const answered = [];
|
|
461
617
|
for (const { value, answer } of received) {
|
|
462
|
-
|
|
618
|
+
if (value < paddingOffset)
|
|
619
|
+
continue;
|
|
620
|
+
const targets = duplicates.get(value - paddingOffset);
|
|
621
|
+
const leader = targets[0];
|
|
463
622
|
if (answer === "declined") {
|
|
464
|
-
|
|
623
|
+
const failure = Result.fail(declined);
|
|
624
|
+
for (const index of targets)
|
|
625
|
+
results[index] = failure;
|
|
465
626
|
} else {
|
|
466
|
-
|
|
467
|
-
|
|
627
|
+
const success = Result.succeed(answer);
|
|
628
|
+
for (const index of targets)
|
|
629
|
+
results[index] = success;
|
|
630
|
+
answered.push({ question: questions[leader], print: prints[leader], noul: answer });
|
|
468
631
|
}
|
|
469
632
|
}
|
|
470
|
-
return cache === undefined ? Effect.void : cache.set(state, answered);
|
|
633
|
+
return cache === undefined || answered.length === 0 ? Effect.void : cache.set(state, answered);
|
|
471
634
|
})), { concurrency: "unbounded", discard: true });
|
|
472
635
|
return results;
|
|
473
636
|
});
|
|
@@ -619,7 +782,8 @@ var itemLocation = (item) => ({
|
|
|
619
782
|
...item.path === undefined ? {} : { path: item.path },
|
|
620
783
|
...item.lines === undefined ? {} : { lines: item.lines }
|
|
621
784
|
});
|
|
622
|
-
var
|
|
785
|
+
var compareStrings = (a, b) => a < b ? -1 : a > b ? 1 : 0;
|
|
786
|
+
var byLocation = (a, b) => compareStrings(a.path ?? "", b.path ?? "") || (a.lines?.[0] ?? 0) - (b.lines?.[0] ?? 0) || compareStrings(a.id, b.id);
|
|
623
787
|
var select = Effect4.fn("select")(function* (policy, cwd, runs) {
|
|
624
788
|
return (yield* runSelector(policy, cwd, runs)).toSorted(byLocation);
|
|
625
789
|
});
|
|
@@ -754,7 +918,35 @@ var loadLabels = Effect6.fn("loadLabels")(function* (policyName, directory = ".l
|
|
|
754
918
|
return yield* parseLabels(file, text);
|
|
755
919
|
});
|
|
756
920
|
var overlaps = (lines, [start, end]) => lines[0] <= end && start <= lines[1];
|
|
921
|
+
var overlapLength = ([a0, a1], [b0, b1]) => Math.max(0, Math.min(a1, b1) - Math.max(a0, b0) + 1);
|
|
757
922
|
var itemLabel = (target, item) => target.label !== "violates" || target.lines === undefined ? target.label : target.lines.some((range) => item.lines === undefined || overlaps(item.lines, range)) ? "violates" : "complies";
|
|
923
|
+
var labelFileItems = (target, items) => {
|
|
924
|
+
if (target.label !== "violates" || target.lines === undefined)
|
|
925
|
+
return items.map(() => target.label);
|
|
926
|
+
const ranges = target.lines;
|
|
927
|
+
return items.map((item) => {
|
|
928
|
+
if (item.lines === undefined)
|
|
929
|
+
return "violates";
|
|
930
|
+
const [start, end] = item.lines;
|
|
931
|
+
const matched = ranges.filter((range) => overlaps(item.lines, range));
|
|
932
|
+
if (matched.length === 0)
|
|
933
|
+
return "complies";
|
|
934
|
+
const onlyNested = matched.every((range) => {
|
|
935
|
+
const rangeLen = range[1] - range[0] + 1;
|
|
936
|
+
const mine = overlapLength(item.lines, range);
|
|
937
|
+
if (mine / rangeLen >= 0.5)
|
|
938
|
+
return false;
|
|
939
|
+
return items.some((other) => {
|
|
940
|
+
if (other === item || other.lines === undefined)
|
|
941
|
+
return false;
|
|
942
|
+
const [os, oe] = other.lines;
|
|
943
|
+
const strictlyEncloses = os <= start && end <= oe && (os < start || oe > end);
|
|
944
|
+
return strictlyEncloses && overlapLength(other.lines, range) > mine;
|
|
945
|
+
});
|
|
946
|
+
});
|
|
947
|
+
return onlyNested ? "complies" : "violates";
|
|
948
|
+
});
|
|
949
|
+
};
|
|
758
950
|
var reachedRanges = (ranges, items) => ranges.filter((range) => items.some((item) => item.lines === undefined || overlaps(item.lines, range))).length;
|
|
759
951
|
var targetScore = (scores) => {
|
|
760
952
|
const answered = scores.filter((score) => score !== null);
|
|
@@ -784,6 +976,55 @@ var separation = (low, high) => {
|
|
|
784
976
|
};
|
|
785
977
|
var scoresOf = (cases, label) => cases.filter((scored) => scored.label === label).map(probability);
|
|
786
978
|
var auroc = (cases) => separation(scoresOf(cases, "complies"), scoresOf(cases, "violates"));
|
|
979
|
+
var auprc = (cases) => {
|
|
980
|
+
const clear = cases.filter((scored) => scored.label !== "ambiguous");
|
|
981
|
+
const totalPos = clear.filter((scored) => scored.label === "violates").length;
|
|
982
|
+
if (totalPos === 0 || clear.length === totalPos)
|
|
983
|
+
return Number.NaN;
|
|
984
|
+
const sorted = clear.map((scored) => ({ score: probability(scored), pos: scored.label === "violates" })).sort((a, b) => b.score - a.score);
|
|
985
|
+
let tp = 0;
|
|
986
|
+
let seen = 0;
|
|
987
|
+
let ap = 0;
|
|
988
|
+
for (let start = 0;start < sorted.length; ) {
|
|
989
|
+
let end = start;
|
|
990
|
+
let groupPos = 0;
|
|
991
|
+
while (end < sorted.length && sorted[end].score === sorted[start].score) {
|
|
992
|
+
if (sorted[end].pos)
|
|
993
|
+
groupPos++;
|
|
994
|
+
end++;
|
|
995
|
+
}
|
|
996
|
+
const groupSize = end - start;
|
|
997
|
+
if (groupPos > 0) {
|
|
998
|
+
let sumPrec = 0;
|
|
999
|
+
for (let k = 1;k <= groupPos; k++) {
|
|
1000
|
+
const expectedSeen = seen + k * (groupSize + 1) / (groupPos + 1);
|
|
1001
|
+
sumPrec += (tp + k) / expectedSeen;
|
|
1002
|
+
}
|
|
1003
|
+
ap += sumPrec / totalPos;
|
|
1004
|
+
}
|
|
1005
|
+
tp += groupPos;
|
|
1006
|
+
seen += groupSize;
|
|
1007
|
+
start = end;
|
|
1008
|
+
}
|
|
1009
|
+
return ap;
|
|
1010
|
+
};
|
|
1011
|
+
var tprAtFpr = (cases, fpr) => {
|
|
1012
|
+
const clean = scoresOf(cases, "complies").sort((a, b) => b - a);
|
|
1013
|
+
const viol = scoresOf(cases, "violates");
|
|
1014
|
+
if (clean.length === 0 || viol.length === 0)
|
|
1015
|
+
return Number.NaN;
|
|
1016
|
+
const cut = clean[Math.min(clean.length - 1, Math.floor(clean.length * fpr))];
|
|
1017
|
+
const above = viol.filter((value) => value > cut).length;
|
|
1018
|
+
const tied = viol.filter((value) => value === cut).length;
|
|
1019
|
+
const cleanAbove = clean.filter((value) => value > cut).length;
|
|
1020
|
+
const cleanTied = clean.filter((value) => value === cut).length;
|
|
1021
|
+
const allowedFromTie = cleanTied > 0 ? Math.max(0, Math.min(1, (fpr * clean.length - cleanAbove) / cleanTied)) : 0;
|
|
1022
|
+
return (above + tied * allowedFromTie) / viol.length;
|
|
1023
|
+
};
|
|
1024
|
+
var violationPrecision = (cases, thresholds = cliThresholds) => {
|
|
1025
|
+
const clearFlagged = cases.filter((scored) => scored.label !== "ambiguous" && verdict(probability(scored), thresholds) === "violation");
|
|
1026
|
+
return clearFlagged.length === 0 ? Number.NaN : clearFlagged.filter((scored) => scored.label === "violates").length / clearFlagged.length;
|
|
1027
|
+
};
|
|
787
1028
|
var concordance = (cases) => {
|
|
788
1029
|
const clean = scoresOf(cases, "complies");
|
|
789
1030
|
const ambiguous = scoresOf(cases, "ambiguous");
|
|
@@ -918,7 +1159,10 @@ var scorePolicy = Effect6.fn("scorePolicy")(function* (policy, labels, cwd) {
|
|
|
918
1159
|
}
|
|
919
1160
|
return {
|
|
920
1161
|
targets: ofTarget.map(({ record, items }) => ({ subject: record, label: record.label, group: targetKey(record), score: targetScore(items.map(score)) })),
|
|
921
|
-
items: ofTarget.flatMap(({ record, items }) =>
|
|
1162
|
+
items: ofTarget.flatMap(({ record, items }) => {
|
|
1163
|
+
const itemLabels = labelFileItems(record, items);
|
|
1164
|
+
return items.map((item, index) => ({ subject: item, label: itemLabels[index], group: targetKey(record), score: score(item) }));
|
|
1165
|
+
}),
|
|
922
1166
|
reach
|
|
923
1167
|
};
|
|
924
1168
|
});
|
|
@@ -936,6 +1180,7 @@ export {
|
|
|
936
1180
|
PolicyError,
|
|
937
1181
|
RequestTooLarge,
|
|
938
1182
|
SelectorError,
|
|
1183
|
+
auprc,
|
|
939
1184
|
auroc,
|
|
940
1185
|
bootstrapInterval,
|
|
941
1186
|
brier,
|
|
@@ -946,6 +1191,7 @@ export {
|
|
|
946
1191
|
cliThresholds,
|
|
947
1192
|
concordance,
|
|
948
1193
|
configuredModel,
|
|
1194
|
+
configuredSecondaryTarget,
|
|
949
1195
|
configuredTarget,
|
|
950
1196
|
decide,
|
|
951
1197
|
declinesAll,
|
|
@@ -959,6 +1205,7 @@ export {
|
|
|
959
1205
|
judge,
|
|
960
1206
|
keyHeader,
|
|
961
1207
|
killRunningSelectors,
|
|
1208
|
+
labelFileItems,
|
|
962
1209
|
lint,
|
|
963
1210
|
loadLabels,
|
|
964
1211
|
loadPolicies,
|
|
@@ -978,9 +1225,11 @@ export {
|
|
|
978
1225
|
sumUsage,
|
|
979
1226
|
targetKey,
|
|
980
1227
|
targetScore,
|
|
1228
|
+
tprAtFpr,
|
|
981
1229
|
tuneThresholds,
|
|
982
1230
|
verdict,
|
|
983
1231
|
verdictF1,
|
|
984
1232
|
verdictOf,
|
|
1233
|
+
violationPrecision,
|
|
985
1234
|
workersAiTarget
|
|
986
1235
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Cause, Context, Effect, Layer, Redacted, Result, Runtime, Schema } from "effect";
|
|
1
|
+
import { Cause, Context, Effect, Layer, Option, Redacted, Result, Runtime, Schema } from "effect";
|
|
2
2
|
import { type MediaKind } from "./media.ts";
|
|
3
3
|
/**
|
|
4
4
|
* One yes/no question: `instructions` is what the decision model judges, and `criteria` say what a yes
|
|
@@ -138,6 +138,12 @@ export declare const configuredModel: Effect.Effect<Model, DecisionModelError>;
|
|
|
138
138
|
* `CLOUDFLARE_AUTH_TOKEN`, and optionally `CLOUDFLARE_API_BASE_URL`.
|
|
139
139
|
*/
|
|
140
140
|
export declare const configuredTarget: Effect.Effect<Target, DecisionModelError>;
|
|
141
|
+
/**
|
|
142
|
+
* The optional secondary decision model the environment configures for cascading (`LINT_ANYTHING_SECONDARY_ENDPOINT`,
|
|
143
|
+
* `LINT_ANYTHING_SECONDARY_MODEL`, `LINT_ANYTHING_SECONDARY_API_KEY`). Returns `Option.none()` when unset.
|
|
144
|
+
* Setting `LINT_ANYTHING_SECONDARY_MODEL` alone (for example `clef`) asks that Cloudflare model on Workers AI.
|
|
145
|
+
*/
|
|
146
|
+
export declare const configuredSecondaryTarget: Effect.Effect<Option.Option<Target>, DecisionModelError>;
|
|
141
147
|
/**
|
|
142
148
|
* One request: its exact body, the media its questions show (by path), and what each question id answers.
|
|
143
149
|
* The body of a request with media leaves them out; they are read, and put where its format takes them,
|
|
@@ -177,6 +183,15 @@ export interface Usage {
|
|
|
177
183
|
}
|
|
178
184
|
export declare const noUsage: Usage;
|
|
179
185
|
export declare const sumUsage: (usages: Iterable<Usage>) => Usage;
|
|
186
|
+
/** Configuration for cascading two decision models: a fast primary and a secondary tiebreaker. */
|
|
187
|
+
export interface CascadeOptions {
|
|
188
|
+
/** Primary probabilities below `low` are accepted directly without calling `secondary` (default: 0.10). */
|
|
189
|
+
readonly low?: number;
|
|
190
|
+
/** Primary probabilities above `high` are accepted directly without calling `secondary` (default: 0.90). */
|
|
191
|
+
readonly high?: number;
|
|
192
|
+
/** Combines primary and secondary probabilities for items sent to both (default: arithmetic mean). */
|
|
193
|
+
readonly combine?: (primary: number, secondary: number) => number;
|
|
194
|
+
}
|
|
180
195
|
declare const DecisionModel_base: Context.ServiceClass<DecisionModel, "lint-anything/DecisionModel", {
|
|
181
196
|
/** The model asked, and where. */
|
|
182
197
|
readonly target: Target;
|
|
@@ -190,12 +205,19 @@ declare const DecisionModel_base: Context.ServiceClass<DecisionModel, "lint-anyt
|
|
|
190
205
|
}>;
|
|
191
206
|
export declare class DecisionModel extends DecisionModel_base {
|
|
192
207
|
/**
|
|
193
|
-
* The decision model the environment configures (`configuredTarget`): Clef on Workers AI by default
|
|
194
|
-
*
|
|
195
|
-
*
|
|
208
|
+
* The decision model the environment configures (`configuredTarget`): Clef on Workers AI by default, or
|
|
209
|
+
* a cross-family cascade when `LINT_ANYTHING_SECONDARY_*` (`configuredSecondaryTarget`) is also set.
|
|
210
|
+
* With `cacheDirectory`, answers are cached in a database there per question across runs, and answers
|
|
211
|
+
* older than `cacheMaxAgeMs` are deleted and asked again.
|
|
196
212
|
*/
|
|
197
213
|
static readonly layer: (cacheDirectory?: string) => Layer.Layer<DecisionModel, DecisionModelError, never>;
|
|
198
214
|
/** `target`, whatever the environment says; answers are cached as with `layer`. */
|
|
199
215
|
static readonly layerFor: (target: Target, cacheDirectory?: string) => Layer.Layer<DecisionModel, DecisionModelError, never>;
|
|
216
|
+
/**
|
|
217
|
+
* Cross-family cascade: asks `primary` first, and asks `secondary` only for questions whose primary
|
|
218
|
+
* probability lands in `[low, high]`, combining both probabilities with `combine` (default: mean).
|
|
219
|
+
* Combines `stats` from both models.
|
|
220
|
+
*/
|
|
221
|
+
static readonly cascadeLayer: (primary: Target, secondary: Target, cacheDirectory?: string, options?: CascadeOptions) => Layer.Layer<DecisionModel, DecisionModelError, never>;
|
|
200
222
|
}
|
|
201
223
|
export {};
|
package/dist/types/lib.d.ts
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* with the metrics and bootstrap intervals the evals use; the skill's score script is built on it.
|
|
19
19
|
*/
|
|
20
20
|
import { DecisionModel, DecisionModelError } from "./decision-model.ts";
|
|
21
|
-
export { cacheMaxAgeMs, clef, clefInputTokenPrice, configuredModel, configuredTarget, Declined, declinesAll, decodeReply, DecisionModel, DecisionModelError, endpointTarget, type Format, keyHeader, type Model, noUsage, planRequests, type Planned, type Question, RequestTooLarge, sumUsage, type Target, type Unanswered, type Usage, workersAiTarget } from "./decision-model.ts";
|
|
21
|
+
export { cacheMaxAgeMs, type CascadeOptions, clef, clefInputTokenPrice, configuredModel, configuredSecondaryTarget, configuredTarget, Declined, declinesAll, decodeReply, DecisionModel, DecisionModelError, endpointTarget, type Format, keyHeader, type Model, noUsage, planRequests, type Planned, type Question, RequestTooLarge, sumUsage, type Target, type Unanswered, type Usage, workersAiTarget } from "./decision-model.ts";
|
|
22
22
|
/** @deprecated Renamed `DecisionModel`: it asks whichever decision model is configured, Clef by default. */
|
|
23
23
|
export declare const Clef: typeof DecisionModel;
|
|
24
24
|
/** @deprecated Renamed `DecisionModel`. */
|
|
@@ -31,4 +31,4 @@ export { type Item, parseItem } from "./item.ts";
|
|
|
31
31
|
export { decide, defaultThreshold, type Finding, itemQuestion, judge, lint, reviewFloor, select, type Verdict, verdictOf } from "./lint.ts";
|
|
32
32
|
export { loadPolicies, parsePolicy, type Policy, PolicyError } from "./policy.ts";
|
|
33
33
|
export { killRunningSelectors, runSelector, SelectorError, type SelectorRuns, selectorTimeoutMinutes } from "./select.ts";
|
|
34
|
-
export { auroc, bootstrapInterval, brier, caught, cliThresholds, concordance, falseAlarms, itemLabel, itemsOf, type Label, LabelError, type LabelRecord, loadLabels, pairedInterval, parseLabels, type PolicyScore, type Quality, quality, type Range, reachedRanges, type Scored, scorePolicy, type ScoreVerdict, type Split, targetKey, targetScore, type Thresholds, tuneThresholds, verdict, verdictF1 } from "./score.ts";
|
|
34
|
+
export { auprc, auroc, bootstrapInterval, brier, caught, cliThresholds, concordance, falseAlarms, itemLabel, itemsOf, type Label, LabelError, labelFileItems, type LabelRecord, loadLabels, pairedInterval, parseLabels, type PolicyScore, type Quality, quality, type Range, reachedRanges, type Scored, scorePolicy, type ScoreVerdict, type Split, targetKey, targetScore, type Thresholds, tprAtFpr, tuneThresholds, verdict, verdictF1, violationPrecision } from "./score.ts";
|
package/dist/types/score.d.ts
CHANGED
|
@@ -56,6 +56,16 @@ export declare const itemLabel: (target: {
|
|
|
56
56
|
readonly label: Label;
|
|
57
57
|
readonly lines?: ReadonlyArray<Range>;
|
|
58
58
|
}, item: Item) => Label;
|
|
59
|
+
/**
|
|
60
|
+
* Labels every item emitted for one target together, preventing nested-range artifacts: when a violating
|
|
61
|
+
* target emits both an enclosing unit and a smaller item strictly nested inside it (e.g. a clean helper
|
|
62
|
+
* closure inside a 100-line violating function), the nested item stays `complies` if every gold range it
|
|
63
|
+
* overlaps is covered < 50% by it and strictly better covered by an enclosing emitted item.
|
|
64
|
+
*/
|
|
65
|
+
export declare const labelFileItems: (target: {
|
|
66
|
+
readonly label: Label;
|
|
67
|
+
readonly lines?: ReadonlyArray<Range>;
|
|
68
|
+
}, items: ReadonlyArray<Item>) => Array<Label>;
|
|
59
69
|
/** How many of `ranges` some item covers; items without lines cover every range. */
|
|
60
70
|
export declare const reachedRanges: (ranges: ReadonlyArray<Range>, items: ReadonlyArray<Item>) => number;
|
|
61
71
|
/**
|
|
@@ -92,6 +102,18 @@ export type ScoreVerdict = "pass" | "review" | "violation";
|
|
|
92
102
|
export declare const verdict: (score: number, { floor, threshold }: Thresholds) => ScoreVerdict;
|
|
93
103
|
/** Separation of clear violations from clean units (ambiguous units excluded). */
|
|
94
104
|
export declare const auroc: (cases: ReadonlyArray<Scored<unknown>>) => number;
|
|
105
|
+
/**
|
|
106
|
+
* Area under the precision-recall curve (Average Precision) on clear cases (ambiguous excluded). Ties are
|
|
107
|
+
* handled by grouping equal scores so ordering within ties cannot inflate or deflate precision.
|
|
108
|
+
*/
|
|
109
|
+
export declare const auprc: (cases: ReadonlyArray<Scored<unknown>>) => number;
|
|
110
|
+
/**
|
|
111
|
+
* True positive rate at a target false-positive rate `fpr` on clear cases: share of violations scored
|
|
112
|
+
* strictly above the `(1 - fpr)` clean quantile (with linear handling of boundary ties).
|
|
113
|
+
*/
|
|
114
|
+
export declare const tprAtFpr: (cases: ReadonlyArray<Scored<unknown>>, fpr: number) => number;
|
|
115
|
+
/** Precision of `violation` findings (`score >= threshold`) on clear cases (ambiguous excluded). */
|
|
116
|
+
export declare const violationPrecision: (cases: ReadonlyArray<Scored<unknown>>, thresholds?: Thresholds) => number;
|
|
95
117
|
/**
|
|
96
118
|
* Ordinal concordance: over every pair of units with different targets (clean < ambiguous < violates),
|
|
97
119
|
* the share where the higher target gets the higher score. Rewards placing ambiguous units between.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@andrueandersoncs/lint-anything",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Check anything against plain-English policies, judged by the decision model of your choice: any System One, OpenAI Decisions, or Gemini Decisions endpoint, or Workers AI",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Andrue Anderson",
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
"access": "public"
|
|
34
34
|
},
|
|
35
35
|
"scripts": {
|
|
36
|
-
"build": "bun build index.ts --target=bun --minify --outfile dist/cli.js && bun build score.ts --target=bun --minify --outfile skills/lint-anything/scripts/score.js && bun build src/lib.ts --target=bun --external effect --external 'effect/*' --outfile dist/lib.js && rm -rf dist/types && tsc -p tsconfig.lib.json",
|
|
36
|
+
"build": "bun build index.ts --target=bun --minify --outfile dist/cli.js && bun build score.ts --target=bun --minify --outfile skills/lint-anything/scripts/score.js && bun build select-steps.ts --target=bun --minify --outfile skills/lint-anything/scripts/select.js && bun build src/lib.ts --target=bun --external effect --external 'effect/*' --outfile dist/lib.js && rm -rf dist/types && tsc -p tsconfig.lib.json",
|
|
37
37
|
"prepublishOnly": "bunx tsc --noEmit && bun run build"
|
|
38
38
|
},
|
|
39
39
|
"peerDependencies": {
|
|
@@ -24,6 +24,7 @@ If `lint-anything` isn't found, install it with `bun add -d @andrueandersoncs/li
|
|
|
24
24
|
- **The working directory matters.** Policies are read from `./.lint-anything/rules`, and selectors run in the current directory. Run from a subdirectory and you get `error: cannot read policies in .lint-anything/rules`: there is no upward search.
|
|
25
25
|
- **The decision model** is the project's choice, set in the environment or in `.env` / `.env.local` in the current directory. Don't change it to make findings go away. With `LINT_ANYTHING_ENDPOINT` (a full URL), the model named by `LINT_ANYTHING_MODEL` is asked there, and `LINT_ANYTHING_API_KEY` is its key if it needs one. An endpoint on `generativelanguage.googleapis.com` speaks Google's Decisions API (`gemini-decision-1`, keyed by a Gemini API key); any other whose path ends in `/decisions` speaks OpenAI's; any other speaks System One (`/v1/systemone`: Jev, LiteLLM, LLM Gateway, and most decision-model servers).
|
|
26
26
|
- **Without an endpoint**, a Cloudflare model on Workers AI is asked: `clef`, or another that `LINT_ANYTHING_MODEL` names (`clef-flash`). `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_AUTH_TOKEN` are then required on every run, even when every answer would come from the cache. `CLOUDFLARE_API_BASE_URL` is optional.
|
|
27
|
+
- **Optional cross-family cascading**: setting `LINT_ANYTHING_SECONDARY_MODEL` (plus optionally `LINT_ANYTHING_SECONDARY_ENDPOINT` and `LINT_ANYTHING_SECONDARY_API_KEY`; or `LINT_ANYTHING_SECONDARY_MODEL=clef` alone for Workers AI) asks the primary model first and escalates only uncertain items in `[0.10, 0.90]` to the secondary model, averaging both answers. On the 216-file unseen holdout (H32/H34), `Gemini Decision 1 → Clef` cut false alarms nearly in half (`12.8% → 6.8%`) and cost less than single-model Clef while sharing the same SQLite cache.
|
|
27
28
|
- **What the model can see** besides text depends on it: Gemini Decision 1 and `clef-omni` take images, audio, and video; OpenAI Decisions, `clef`, and `clef-flash` take images only; another System One server takes what it takes. An item whose media the model can't take stops the run before anything is sent (`Clef takes text and images, not audio clips: …`).
|
|
28
29
|
- **Accuracy numbers in this skill were measured on Clef**, on text. A project on another model, or with media, should score its policies on its labels before trusting them.
|
|
29
30
|
- **Add `.lint-anything/cache` to `.gitignore`.** The answer cache (`answers.sqlite` plus `-wal`/`-shm` files) is written there.
|
|
@@ -124,8 +125,8 @@ Frontmatter keys are `select` and `criteria` (with both `true` and `false`), bot
|
|
|
124
125
|
|
|
125
126
|
The model's "yes" is read as **"this item violates the rule"**:
|
|
126
127
|
|
|
127
|
-
- `criteria.true` describes an item that **violates** the rule.
|
|
128
|
-
- `criteria.false` describes an item that **follows** it, **or
|
|
128
|
+
- `criteria.true` describes an item that **violates** the rule — be rule-specific rather than generic ("The item contains a violation"). On the fresh 216-file holdout (H28/H34), replacing generic criteria with rule-specific `criteria.true` and exemption-explicit `criteria.false` plus targeted selector fixes raised Applicable File AUROC `+0.015`, Localized Caught `+6.5 pts`, and TPR@5% `+6.5 pts` outside noise at the same false-alarm rate.
|
|
129
|
+
- `criteria.false` describes an item that **follows** it, **its explicit exemptions**, **or items the rule doesn't govern**. Selectors usually over-select, and decision models read `criteria` immediately adjacent to every question.
|
|
129
130
|
|
|
130
131
|
If you swap them, every verdict inverts. Nothing catches this but the evidence: before relying on a new policy, run it on at least one known violation and one known-clean item and check that the scores point the right way.
|
|
131
132
|
|
|
@@ -198,7 +199,19 @@ rg --json -U -P '^(?:[ \t]*(?://|#).*\n)+.*' -g '*.{ts,py,go}' |
|
|
|
198
199
|
lines: [.data.line_number, .data.line_number + ($t | split("\n") | length) - 1], text: $t}'
|
|
199
200
|
```
|
|
200
201
|
|
|
201
|
-
Structural units with
|
|
202
|
+
Structural units with the bundled selector helper (`scripts/select.js`, run `--help` to see every step and flag):
|
|
203
|
+
|
|
204
|
+
```yaml
|
|
205
|
+
select: |
|
|
206
|
+
rg --files -g '*.{ts,tsx,js,py,go}' |
|
|
207
|
+
bun <this skill's directory>/scripts/select.js functions --flags --no-call-items |
|
|
208
|
+
bun <this skill's directory>/scripts/select.js file-role |
|
|
209
|
+
bun <this skill's directory>/scripts/select.js present --no-lines
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
`scripts/select.js` bundles the eval-proven file cutters (`functions`, `comments`, `errors`, `errors-go`, `literals`, `module-exports`, `module-state`, `module-outline`, `tests`, `duplicates`, `excerpts`, `whole-file`) and pipe steps (`file-role`, `enclosing`, `blame`, `present`). Steps that parse ASTs resolve `ast-grep` from `PATH` or `./node_modules/.bin/ast-grep` (install with `bun add -d @ast-grep/cli`). Copy `scripts/select.js` into `.lint-anything/lib/select.js` (or reference it from `node_modules/@andrueandersoncs/lint-anything/skills/lint-anything/scripts/select.js`) so CI and teammates share the exact same selector path. Pass `--no-lines` to `present` to omit `(lines start-end)` from `text` and preserve cache hits when lines above shift.
|
|
213
|
+
|
|
214
|
+
Or write a custom ast-grep selector directly:
|
|
202
215
|
|
|
203
216
|
```sh
|
|
204
217
|
ast-grep run -p 'function $NAME($$$PARAMS) { $$$BODY }' -l ts --json=stream |
|
|
@@ -265,7 +278,7 @@ One JSON object per line, one per labeled file:
|
|
|
265
278
|
- Where they disagree, or any of them calls it ambiguous, **ask the user** about that file, showing the rule, the code, and each model's reasoning. Record what the user decides; if the user says reasonable reviewers could disagree, record `ambiguous`. Ambiguous files are left out of AUROC and counted as right when they land in `review`.
|
|
266
279
|
- One model alone labels with its own reading of the rule, and tuning against those labels only makes the policy agree with that reading.
|
|
267
280
|
|
|
268
|
-
The lint-anything repository has worked examples: 21 policies in `evals/corpus/policies/` and the
|
|
281
|
+
The lint-anything repository has worked examples: 21 policies in `evals/corpus/policies/` and the source of the bundled `scripts/select.js` helpers in `evals/steps/`. Its evals score them the same way, against a labeled corpus. `$EVAL_STEPS` exists only inside those evals; in real projects use `scripts/select.js` (or copy it into `.lint-anything/lib/`).
|
|
269
282
|
|
|
270
283
|
## Cache and cost
|
|
271
284
|
|