@andrueandersoncs/lint-anything 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -8
- package/dist/cli.js +66 -66
- package/dist/lib.js +390 -32
- package/dist/types/decision-model.d.ts +43 -6
- package/dist/types/item.d.ts +4 -1
- package/dist/types/lib.d.ts +2 -2
- package/dist/types/lint.d.ts +2 -0
- package/dist/types/media.d.ts +30 -0
- package/dist/types/score.d.ts +24 -0
- package/dist/types/select.d.ts +3 -2
- package/package.json +1 -1
- package/skills/lint-anything/SKILL.md +20 -7
- package/skills/lint-anything/scripts/score.js +43 -43
package/dist/lib.js
CHANGED
|
@@ -6,6 +6,40 @@ import { join } from "path";
|
|
|
6
6
|
import { Cause, Config, Context, Effect, flow, Layer, Option, Redacted, Ref, Result, Runtime, Schedule, Schema, Semaphore } from "effect";
|
|
7
7
|
import { FetchHttpClient, HttpClient, HttpClientError, HttpClientRequest } from "effect/http";
|
|
8
8
|
|
|
9
|
+
// src/media.ts
|
|
10
|
+
import { extname } from "path";
|
|
11
|
+
var mediaKinds = ["image", "audio", "video"];
|
|
12
|
+
var mediaNoun = { image: "images", audio: "audio clips", video: "videos" };
|
|
13
|
+
var types = {
|
|
14
|
+
".png": { type: "image/png", kind: "image" },
|
|
15
|
+
".jpg": { type: "image/jpeg", kind: "image" },
|
|
16
|
+
".jpeg": { type: "image/jpeg", kind: "image" },
|
|
17
|
+
".webp": { type: "image/webp", kind: "image" },
|
|
18
|
+
".wav": { type: "audio/wav", kind: "audio" },
|
|
19
|
+
".mp3": { type: "audio/mpeg", kind: "audio" },
|
|
20
|
+
".flac": { type: "audio/flac", kind: "audio" },
|
|
21
|
+
".ogg": { type: "audio/ogg", kind: "audio" },
|
|
22
|
+
".mp4": { type: "video/mp4", kind: "video" },
|
|
23
|
+
".webm": { type: "video/webm", kind: "video" }
|
|
24
|
+
};
|
|
25
|
+
var mediaExtensions = Object.keys(types);
|
|
26
|
+
var mediaTypeOf = (path) => types[extname(path).toLowerCase()];
|
|
27
|
+
var maxMediaFiles = { image: 4, audio: 4, video: 2 };
|
|
28
|
+
var maxImageBytes = 4 * 1024 * 1024;
|
|
29
|
+
var maxMediaBytes = 8 * 1024 * 1024;
|
|
30
|
+
var readMedium = async (path) => {
|
|
31
|
+
const { type, kind } = mediaTypeOf(path);
|
|
32
|
+
const hasher = new Bun.CryptoHasher("sha256");
|
|
33
|
+
let bytes = 0;
|
|
34
|
+
for await (const chunk of Bun.file(path).stream()) {
|
|
35
|
+
hasher.update(chunk);
|
|
36
|
+
bytes += chunk.byteLength;
|
|
37
|
+
}
|
|
38
|
+
return { path, type, kind, bytes, digest: hasher.digest("hex") };
|
|
39
|
+
};
|
|
40
|
+
var mediaBase64 = async (medium) => Buffer.from(await Bun.file(medium.path).bytes()).toString("base64");
|
|
41
|
+
|
|
42
|
+
// src/decision-model.ts
|
|
9
43
|
class DecisionModelError extends Schema.TaggedError()("DecisionModelError", {
|
|
10
44
|
message: Schema.String
|
|
11
45
|
}) {
|
|
@@ -32,6 +66,7 @@ var attemptRetries = 3;
|
|
|
32
66
|
var cacheMaxAgeMs = 30 * 24 * 60 * 60 * 1000;
|
|
33
67
|
var cacheFile = "answers.sqlite";
|
|
34
68
|
var cacheBusyTimeoutMs = 1e4;
|
|
69
|
+
var dataUrl = (medium) => `data:${medium.type};base64,${medium.base64}`;
|
|
35
70
|
var bearer = (key) => ["authorization", `Bearer ${key}`];
|
|
36
71
|
var never = () => false;
|
|
37
72
|
var Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 }));
|
|
@@ -41,8 +76,18 @@ var SystemOneResponse = Schema.Struct({
|
|
|
41
76
|
answers: Schema.Record(Schema.String, Schema.Struct({ noul: Probability })),
|
|
42
77
|
usage: ReportedUsage
|
|
43
78
|
});
|
|
79
|
+
var systemOneMedia = (media) => {
|
|
80
|
+
const of = (kind) => media.filter((medium) => medium.kind === kind).map(dataUrl);
|
|
81
|
+
const [images, audio, videos] = [of("image"), of("audio"), of("video")];
|
|
82
|
+
return {
|
|
83
|
+
...images.length === 0 ? {} : { images },
|
|
84
|
+
...audio.length === 0 ? {} : { audio },
|
|
85
|
+
...videos.length === 0 ? {} : { videos }
|
|
86
|
+
};
|
|
87
|
+
};
|
|
44
88
|
var systemOne = {
|
|
45
|
-
head: (model, state) => `${JSON.stringify({ model, state }).slice(0, -1)},"questions":{`,
|
|
89
|
+
head: (model, state, media) => `${JSON.stringify({ model, ...systemOneMedia(media), state }).slice(0, -1)},"questions":{`,
|
|
90
|
+
takes: mediaKinds,
|
|
46
91
|
tail: "}}",
|
|
47
92
|
entry: (id, question) => `${JSON.stringify(id)}:${JSON.stringify(noul(question))}`,
|
|
48
93
|
decode: (body) => Schema.decodeUnknownEffect(SystemOneResponse)(typeof body === "object" && body !== null && "result" in body ? body.result : body).pipe(Effect.map(({ answers, usage }) => ({
|
|
@@ -67,8 +112,16 @@ var DecisionsResponse = Schema.Struct({
|
|
|
67
112
|
])),
|
|
68
113
|
usage: ReportedUsage
|
|
69
114
|
});
|
|
115
|
+
var decisionsInput = (state, media) => media.length === 0 ? state : [{
|
|
116
|
+
role: "user",
|
|
117
|
+
content: [
|
|
118
|
+
...media.map((medium) => ({ type: "input_image", image_url: dataUrl(medium) })),
|
|
119
|
+
{ type: "input_text", text: state }
|
|
120
|
+
]
|
|
121
|
+
}];
|
|
70
122
|
var decisions = {
|
|
71
|
-
head: (model, state) => `${JSON.stringify({ model, input: state }).slice(0, -1)},"questions":[`,
|
|
123
|
+
head: (model, state, media) => `${JSON.stringify({ model, input: decisionsInput(state, media) }).slice(0, -1)},"questions":[`,
|
|
124
|
+
takes: ["image"],
|
|
72
125
|
tail: "]}",
|
|
73
126
|
entry: (id, question) => JSON.stringify(predicate(id, question)),
|
|
74
127
|
decode: (body) => Schema.decodeUnknownEffect(DecisionsResponse)(body).pipe(Effect.map(({ answers, usage }) => ({
|
|
@@ -90,8 +143,13 @@ var GeminiResponse = Schema.Struct({
|
|
|
90
143
|
var GeminiBlocked = Schema.fromJsonString(Schema.Struct({
|
|
91
144
|
error: Schema.Struct({ code: Schema.Literals(["safety", "prohibited_content", "blocklist", "image_safety", "content_blocked"]) })
|
|
92
145
|
}));
|
|
146
|
+
var geminiInput = (state, media) => media.length === 0 ? state : [
|
|
147
|
+
...media.map((medium) => ({ type: medium.kind, data: medium.base64, mime_type: medium.type })),
|
|
148
|
+
{ type: "text", text: state }
|
|
149
|
+
];
|
|
93
150
|
var gemini = {
|
|
94
|
-
head: (model, state) => `${JSON.stringify({ model, input: state }).slice(0, -1)},"questions":{`,
|
|
151
|
+
head: (model, state, media) => `${JSON.stringify({ model, input: geminiInput(state, media) }).slice(0, -1)},"questions":{`,
|
|
152
|
+
takes: mediaKinds,
|
|
95
153
|
tail: "}}",
|
|
96
154
|
entry: (id, question) => `${JSON.stringify(id)}:${JSON.stringify(geminiNoul(question))}`,
|
|
97
155
|
decode: (body) => Schema.decodeUnknownEffect(GeminiResponse)(body).pipe(Effect.map(({ answers, usage }) => ({
|
|
@@ -105,6 +163,11 @@ var wires = { "system-one": systemOne, decisions, gemini };
|
|
|
105
163
|
var decodeReply = (format, body) => wires[format].decode(body);
|
|
106
164
|
var keyHeader = (format, key) => wires[format].keyHeader(key);
|
|
107
165
|
var declinesAll = (format, status, body) => wires[format].declinesAll(status, body);
|
|
166
|
+
var workersAiMedia = {
|
|
167
|
+
clef: ["image"],
|
|
168
|
+
"clef-flash": ["image"],
|
|
169
|
+
"clef-omni": mediaKinds
|
|
170
|
+
};
|
|
108
171
|
var clef = workersAiModel("clef");
|
|
109
172
|
function workersAiModel(model) {
|
|
110
173
|
return {
|
|
@@ -112,7 +175,8 @@ function workersAiModel(model) {
|
|
|
112
175
|
model,
|
|
113
176
|
name: model === "clef" ? "Clef" : model,
|
|
114
177
|
cacheKey: model,
|
|
115
|
-
inputTokenPrice: model === "clef" ? clefInputTokenPrice : undefined
|
|
178
|
+
inputTokenPrice: model === "clef" ? clefInputTokenPrice : undefined,
|
|
179
|
+
media: workersAiMedia[model]
|
|
116
180
|
};
|
|
117
181
|
}
|
|
118
182
|
var workersAiTarget = (options) => {
|
|
@@ -135,7 +199,8 @@ var endpointModel = (url, model) => ({
|
|
|
135
199
|
model,
|
|
136
200
|
name: model,
|
|
137
201
|
cacheKey: `${url} ${model}`,
|
|
138
|
-
inputTokenPrice: undefined
|
|
202
|
+
inputTokenPrice: undefined,
|
|
203
|
+
media: undefined
|
|
139
204
|
});
|
|
140
205
|
var endpointTarget = (options) => ({ ...endpointModel(options.url, options.model), url: options.url, token: options.token });
|
|
141
206
|
var configError = (message) => new DecisionModelError({ message });
|
|
@@ -174,9 +239,9 @@ var configuredTarget = Effect.gen(function* () {
|
|
|
174
239
|
const baseUrl = yield* Config.String("CLOUDFLARE_API_BASE_URL").pipe(Config.withDefault(defaultCloudflareBaseUrl), Effect.mapError((error) => configError(`CLOUDFLARE_API_BASE_URL: ${error.message}`)));
|
|
175
240
|
return workersAiTarget({ accountId, token, model: chosen.model, baseUrl });
|
|
176
241
|
});
|
|
177
|
-
var pack = (model, state, values, question) => {
|
|
242
|
+
var pack = (model, state, media, values, question) => {
|
|
178
243
|
const wire = wires[model.format];
|
|
179
|
-
const head = wire.head(model.model, state);
|
|
244
|
+
const head = wire.head(model.model, state, []);
|
|
180
245
|
const fixedBytes = Buffer.byteLength(head) + Buffer.byteLength(wire.tail);
|
|
181
246
|
const entry = (value, index) => {
|
|
182
247
|
const id = `i${index}`;
|
|
@@ -190,7 +255,7 @@ var pack = (model, state, values, question) => {
|
|
|
190
255
|
let bytes = fixedBytes;
|
|
191
256
|
const flush = () => {
|
|
192
257
|
if (questions.length > 0)
|
|
193
|
-
requests.push({ body: `${head}${entries.join(",")}${wire.tail}`, questions });
|
|
258
|
+
requests.push({ body: `${head}${entries.join(",")}${wire.tail}`, media, questions });
|
|
194
259
|
entries = [];
|
|
195
260
|
questions = [];
|
|
196
261
|
bytes = fixedBytes;
|
|
@@ -213,7 +278,31 @@ var pack = (model, state, values, question) => {
|
|
|
213
278
|
flush();
|
|
214
279
|
return { requests, oversized };
|
|
215
280
|
};
|
|
216
|
-
var planRequests = (state, questions, model = clef) =>
|
|
281
|
+
var planRequests = (state, questions, model = clef) => {
|
|
282
|
+
const groups = new Map;
|
|
283
|
+
questions.forEach((question, index) => {
|
|
284
|
+
const key = JSON.stringify(question.media ?? []);
|
|
285
|
+
const group = groups.get(key);
|
|
286
|
+
if (group === undefined)
|
|
287
|
+
groups.set(key, [index]);
|
|
288
|
+
else
|
|
289
|
+
group.push(index);
|
|
290
|
+
});
|
|
291
|
+
const requests = [];
|
|
292
|
+
const oversized = [];
|
|
293
|
+
for (const indexes of groups.values()) {
|
|
294
|
+
const media = questions[indexes[0]].media ?? [];
|
|
295
|
+
const planned = pack(model, state, media, indexes, (index) => questions[index]);
|
|
296
|
+
requests.push(...planned.requests);
|
|
297
|
+
oversized.push(...planned.oversized);
|
|
298
|
+
}
|
|
299
|
+
return { requests, oversized };
|
|
300
|
+
};
|
|
301
|
+
var withMedia = (model, state, body, media) => {
|
|
302
|
+
const head = wires[model.format].head;
|
|
303
|
+
return `${head(model.model, state, media)}${body.slice(head(model.model, state, []).length)}`;
|
|
304
|
+
};
|
|
305
|
+
var mediaPrint = (media) => media.map((medium) => `${medium.type} ${medium.digest}`).join("\x00");
|
|
217
306
|
var openAnswerCache = (directory, now, model) => {
|
|
218
307
|
mkdirSync(directory, { recursive: true });
|
|
219
308
|
const db = new Database(join(directory, cacheFile));
|
|
@@ -228,15 +317,15 @@ var openAnswerCache = (directory, now, model) => {
|
|
|
228
317
|
db.run("DELETE FROM answers WHERE answered <= ?", [now - cacheMaxAgeMs]);
|
|
229
318
|
const select = db.query("SELECT noul FROM answers WHERE key = ?");
|
|
230
319
|
const upsert = db.query("INSERT OR REPLACE INTO answers (key, noul, answered) VALUES (?, ?, ?)");
|
|
231
|
-
const key = (state, question) => new Bun.CryptoHasher("sha256").update(`${model.cacheKey}\x00${state}\x00${JSON.stringify(noul(question))}`).digest();
|
|
320
|
+
const key = (state, question, print) => new Bun.CryptoHasher("sha256").update(`${model.cacheKey}\x00${state}\x00${JSON.stringify(noul(question))}${print === "" ? "" : `\x00${print}`}`).digest();
|
|
232
321
|
const insert = db.transaction((state, answers) => {
|
|
233
322
|
const answered = Date.now();
|
|
234
|
-
for (const { question, noul } of answers)
|
|
235
|
-
upsert.run(key(state, question), noul, answered);
|
|
323
|
+
for (const { question, print, noul } of answers)
|
|
324
|
+
upsert.run(key(state, question, print), noul, answered);
|
|
236
325
|
});
|
|
237
326
|
const failed = (error) => new DecisionModelError({ message: `answer cache ${join(directory, cacheFile)}: ${error instanceof Error ? error.message : String(error)}` });
|
|
238
327
|
return {
|
|
239
|
-
get: (state, questions) => Effect.try({ try: () => questions.map((question) => select.get(key(state, question))?.noul), catch: failed }),
|
|
328
|
+
get: (state, questions, prints) => Effect.try({ try: () => questions.map((question, index) => select.get(key(state, question, prints[index]))?.noul), catch: failed }),
|
|
240
329
|
set: (state, answers) => Effect.try({ try: () => insert(state, answers), catch: failed }),
|
|
241
330
|
close: () => db.close()
|
|
242
331
|
};
|
|
@@ -269,15 +358,74 @@ var failureMessage = (name, error) => {
|
|
|
269
358
|
const detail = (error instanceof Error ? error.message : String(error)).replace(/\s*\n\s*/g, " ");
|
|
270
359
|
return Effect.succeed(`${name} sent an unexpected response: ${detail}`);
|
|
271
360
|
};
|
|
361
|
+
var jsonRetryDelayMs = (text) => {
|
|
362
|
+
const match = /"retryDelay":\s*"(\d+(?:\.\d+)?)s"/.exec(text);
|
|
363
|
+
return match === null ? undefined : Math.ceil(Number(match[1]) * 1000);
|
|
364
|
+
};
|
|
272
365
|
|
|
273
366
|
class DecisionModel extends Context.Service()("lint-anything/DecisionModel") {
|
|
274
367
|
static layer = (cacheDirectory) => Layer.effect(DecisionModel, Effect.flatMap(configuredTarget, (target) => make(target, cacheDirectory))).pipe(Layer.provide(FetchHttpClient.layer));
|
|
275
368
|
static layerFor = (target, cacheDirectory) => Layer.effect(DecisionModel, make(target, cacheDirectory)).pipe(Layer.provide(FetchHttpClient.layer));
|
|
369
|
+
static cascadeLayer = (primary, secondary, cacheDirectory, options) => Layer.effect(DecisionModel, makeCascade(primary, secondary, cacheDirectory, options)).pipe(Layer.provide(FetchHttpClient.layer));
|
|
276
370
|
}
|
|
371
|
+
var makeCascade = Effect.fnUntraced(function* (primaryTarget, secondaryTarget, cacheDirectory, options) {
|
|
372
|
+
const low = options?.low ?? 0.15;
|
|
373
|
+
const high = options?.high ?? 0.85;
|
|
374
|
+
const combine = options?.combine ?? ((a, b) => (a + b) / 2);
|
|
375
|
+
const primary = yield* make(primaryTarget, cacheDirectory);
|
|
376
|
+
const secondary = yield* make(secondaryTarget, cacheDirectory);
|
|
377
|
+
return DecisionModel.of({
|
|
378
|
+
target: {
|
|
379
|
+
...primary.target,
|
|
380
|
+
name: `${primary.target.name}+${secondary.target.name}`,
|
|
381
|
+
cacheKey: `${primary.target.cacheKey}+${secondary.target.cacheKey}`,
|
|
382
|
+
inputTokenPrice: undefined
|
|
383
|
+
},
|
|
384
|
+
ask: Effect.fn("DecisionModel.cascade.ask")(function* (source, state, questions) {
|
|
385
|
+
const first = [...yield* primary.ask(source, state, questions)];
|
|
386
|
+
const escalate = [];
|
|
387
|
+
first.forEach((result, index) => {
|
|
388
|
+
if (Result.isSuccess(result) && result.success >= low && result.success <= high) {
|
|
389
|
+
escalate.push(index);
|
|
390
|
+
}
|
|
391
|
+
});
|
|
392
|
+
if (escalate.length === 0)
|
|
393
|
+
return first;
|
|
394
|
+
const second = yield* secondary.ask(source, state, escalate.map((index) => questions[index]));
|
|
395
|
+
escalate.forEach((origIndex, subIndex) => {
|
|
396
|
+
const a = first[origIndex];
|
|
397
|
+
const b = second[subIndex];
|
|
398
|
+
if (Result.isSuccess(a) && Result.isSuccess(b)) {
|
|
399
|
+
first[origIndex] = Result.succeed(combine(a.success, b.success));
|
|
400
|
+
}
|
|
401
|
+
});
|
|
402
|
+
return first;
|
|
403
|
+
}),
|
|
404
|
+
stats: Effect.map(Effect.all([primary.stats, secondary.stats]), ([a, b]) => {
|
|
405
|
+
const merged = new Map(a);
|
|
406
|
+
for (const [source, usage] of b) {
|
|
407
|
+
merged.set(source, sumUsage([merged.get(source) ?? noUsage, usage]));
|
|
408
|
+
}
|
|
409
|
+
return merged;
|
|
410
|
+
})
|
|
411
|
+
});
|
|
412
|
+
});
|
|
277
413
|
var make = Effect.fnUntraced(function* (target, cacheDirectory) {
|
|
278
414
|
const wire = wires[target.format];
|
|
279
415
|
const { token } = target;
|
|
280
|
-
const client = (yield* HttpClient.HttpClient).pipe(HttpClient.mapRequest(flow(token === undefined ? (request) => request : HttpClientRequest.setHeader(...wire.keyHeader(Redacted.value(token))), HttpClientRequest.acceptJson)), HttpClient.filterStatusOk, HttpClient.transformResponse(flow(Effect.tap((response) => response.text), Effect.timeout(attemptTimeout)
|
|
416
|
+
const client = (yield* HttpClient.HttpClient).pipe(HttpClient.mapRequest(flow(token === undefined ? (request) => request : HttpClientRequest.setHeader(...wire.keyHeader(Redacted.value(token))), HttpClientRequest.acceptJson)), HttpClient.filterStatusOk, HttpClient.transformResponse(flow(Effect.tap((response) => response.text), Effect.timeout(attemptTimeout), Effect.tapError((error) => {
|
|
417
|
+
if (!HttpClientError.isHttpClientError(error) || error.reason._tag !== "StatusCodeError")
|
|
418
|
+
return Effect.void;
|
|
419
|
+
const { response } = error.reason;
|
|
420
|
+
if (response.status !== 429 || response.headers["retry-after"] !== undefined)
|
|
421
|
+
return Effect.void;
|
|
422
|
+
return response.text.pipe(Effect.timeout("5 seconds"), Effect.flatMap((text) => {
|
|
423
|
+
const delayMs = jsonRetryDelayMs(text);
|
|
424
|
+
return delayMs !== undefined && delayMs > 0 ? Effect.sleep(delayMs) : Effect.void;
|
|
425
|
+
}), Effect.orElseSucceed(() => {
|
|
426
|
+
return;
|
|
427
|
+
}));
|
|
428
|
+
}))), HttpClient.retryTransient({ schedule: Schedule.exponential(500), times: attemptRetries }));
|
|
281
429
|
const cache = cacheDirectory === undefined ? undefined : yield* Effect.acquireRelease(Effect.try({
|
|
282
430
|
try: () => openAnswerCache(cacheDirectory, Date.now(), target),
|
|
283
431
|
catch: (error) => new DecisionModelError({
|
|
@@ -298,8 +446,38 @@ var make = Effect.fnUntraced(function* (target, cacheDirectory) {
|
|
|
298
446
|
const { response } = error.reason;
|
|
299
447
|
return response.text.pipe(Effect.timeout("5 seconds"), Effect.map((text) => wire.declinesAll(response.status, text)), Effect.orElseSucceed(() => false));
|
|
300
448
|
};
|
|
301
|
-
const
|
|
302
|
-
|
|
449
|
+
const readMedia = (questions) => Effect.forEach([...new Set(questions.flatMap((question) => question.media ?? []))], (path) => Effect.tryPromise({
|
|
450
|
+
try: () => readMedium(path),
|
|
451
|
+
catch: (error) => new DecisionModelError({ message: `cannot read media ${path}: ${error instanceof Error ? error.message : String(error)}` })
|
|
452
|
+
})).pipe(Effect.map((media) => new Map(media.map((medium) => [medium.path, medium]))));
|
|
453
|
+
const takes = (kind) => wire.takes.includes(kind) && (target.media?.includes(kind) ?? true);
|
|
454
|
+
const taken = ["text", ...mediaKinds.filter(takes).map((kind) => mediaNoun[kind])];
|
|
455
|
+
const takenList = taken.length <= 2 ? taken.join(" and ") : `${taken.slice(0, -1).join(", ")}, and ${taken.at(-1)}`;
|
|
456
|
+
const notTaken = (medium) => new DecisionModelError({ message: `${target.name} takes ${takenList}, not ${mediaNoun[medium.kind]}: ${medium.path}` });
|
|
457
|
+
const mediaTooLarge = (media) => {
|
|
458
|
+
const image = media.find((medium) => medium.kind === "image" && medium.bytes > maxImageBytes);
|
|
459
|
+
if (image !== undefined) {
|
|
460
|
+
return new RequestTooLarge({
|
|
461
|
+
bytes: image.bytes,
|
|
462
|
+
message: `image ${image.path} is ${image.bytes} bytes, over the ${maxImageBytes}-byte image limit`
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
const bytes = media.reduce((sum, medium) => sum + medium.bytes, 0);
|
|
466
|
+
return bytes > maxMediaBytes ? new RequestTooLarge({ bytes, message: `media are ${bytes} bytes, over the ${maxMediaBytes}-byte media limit` }) : undefined;
|
|
467
|
+
};
|
|
468
|
+
const bodyOf = (state, request, media) => request.media.length === 0 ? Effect.succeed(request.body) : Effect.tryPromise({
|
|
469
|
+
try: async () => {
|
|
470
|
+
const encoded = await Promise.all(request.media.map(async (path) => {
|
|
471
|
+
const medium = media.get(path);
|
|
472
|
+
return { ...medium, base64: await mediaBase64(medium) };
|
|
473
|
+
}));
|
|
474
|
+
return withMedia(target, state, request.body, encoded);
|
|
475
|
+
},
|
|
476
|
+
catch: (error) => new DecisionModelError({ message: `cannot read media: ${error instanceof Error ? error.message : String(error)}` })
|
|
477
|
+
});
|
|
478
|
+
const send = Effect.fn("DecisionModel.send")(function* (source, state, request, media) {
|
|
479
|
+
const post = (body) => HttpClientRequest.post(target.url).pipe(HttpClientRequest.bodyText(body, "application/json"), client.execute, Effect.flatMap((response) => response.json), Effect.flatMap(wire.decode), Effect.catch((error) => Effect.flatMap(declinesAll(error), (all) => all ? Effect.succeed({ answers: new Map(request.questions.map(({ id }) => [id, "declined"])), inputTokens: undefined }) : Effect.flatMap(failureMessage(target.name, error), (message) => Effect.fail(new DecisionModelError({ message }))))));
|
|
480
|
+
const reply = yield* record(source, { requests: 1 }).pipe(Effect.andThen(bodyOf(state, request, media)), Effect.flatMap(post), permits.withPermit);
|
|
303
481
|
yield* record(source, reply.inputTokens === undefined ? { unmetered: 1 } : { inputTokens: reply.inputTokens });
|
|
304
482
|
const answers = [];
|
|
305
483
|
for (const { id, value } of request.questions) {
|
|
@@ -312,9 +490,12 @@ var make = Effect.fnUntraced(function* (target, cacheDirectory) {
|
|
|
312
490
|
return answers;
|
|
313
491
|
});
|
|
314
492
|
const ask = Effect.fn("DecisionModel.ask")(function* (source, state, questions) {
|
|
493
|
+
const media = yield* readMedia(questions);
|
|
494
|
+
const mediaOf = (question) => (question.media ?? []).map((path) => media.get(path));
|
|
495
|
+
const prints = questions.map((question) => mediaPrint(mediaOf(question)));
|
|
315
496
|
const results = [];
|
|
316
497
|
const misses = [];
|
|
317
|
-
const cached = cache === undefined ? [] : yield* cache.get(state, questions);
|
|
498
|
+
const cached = cache === undefined ? [] : yield* cache.get(state, questions, prints);
|
|
318
499
|
questions.forEach((_, index) => {
|
|
319
500
|
const answer = cached[index];
|
|
320
501
|
if (answer === undefined)
|
|
@@ -324,21 +505,83 @@ var make = Effect.fnUntraced(function* (target, cacheDirectory) {
|
|
|
324
505
|
});
|
|
325
506
|
if (misses.length < questions.length)
|
|
326
507
|
yield* record(source, { cached: questions.length - misses.length });
|
|
327
|
-
const
|
|
328
|
-
for (const
|
|
329
|
-
|
|
330
|
-
|
|
508
|
+
const sendable = [];
|
|
509
|
+
for (const index of misses) {
|
|
510
|
+
const shown = mediaOf(questions[index]);
|
|
511
|
+
const untaken = shown.find((medium) => !takes(medium.kind));
|
|
512
|
+
if (untaken !== undefined)
|
|
513
|
+
return yield* notTaken(untaken);
|
|
514
|
+
const tooMuch = mediaTooLarge(shown);
|
|
515
|
+
if (tooMuch === undefined)
|
|
516
|
+
sendable.push(index);
|
|
517
|
+
else
|
|
518
|
+
results[index] = Result.fail(tooMuch);
|
|
519
|
+
}
|
|
520
|
+
const uniqueSendable = [];
|
|
521
|
+
const duplicates = new Map;
|
|
522
|
+
const byFingerprint = new Map;
|
|
523
|
+
for (const index of sendable) {
|
|
524
|
+
const fingerprint = `${JSON.stringify(noul(questions[index]))}\x00${prints[index]}`;
|
|
525
|
+
const leader = byFingerprint.get(fingerprint);
|
|
526
|
+
if (leader === undefined) {
|
|
527
|
+
const slot = uniqueSendable.length;
|
|
528
|
+
byFingerprint.set(fingerprint, slot);
|
|
529
|
+
uniqueSendable.push(index);
|
|
530
|
+
duplicates.set(slot, [index]);
|
|
531
|
+
} else {
|
|
532
|
+
duplicates.get(leader).push(index);
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
const minWarmBatch = 5;
|
|
536
|
+
const paddingIndices = [];
|
|
537
|
+
if (target.format === "system-one" && uniqueSendable.length > 0 && uniqueSendable.length < minWarmBatch && uniqueSendable.every((index) => (questions[index].media?.length ?? 0) === 0)) {
|
|
538
|
+
const siblingPool = [];
|
|
539
|
+
for (let i = 0;i < questions.length; i++) {
|
|
540
|
+
if (cached[i] !== undefined && (questions[i].media?.length ?? 0) === 0 && questions[i].instructions.length <= 4000) {
|
|
541
|
+
siblingPool.push(i);
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
const needed = minWarmBatch - uniqueSendable.length;
|
|
545
|
+
if (siblingPool.length > 0) {
|
|
546
|
+
const step = Math.max(1, Math.floor(siblingPool.length / needed));
|
|
547
|
+
for (let i = 0;i < siblingPool.length && paddingIndices.length < needed; i += step) {
|
|
548
|
+
paddingIndices.push(siblingPool[i]);
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
let paddingOffset = paddingIndices.length;
|
|
553
|
+
let planned = planRequests(state, [...paddingIndices.map((index) => questions[index]), ...uniqueSendable.map((index) => questions[index])], target);
|
|
554
|
+
if (paddingOffset > 0 && planned.requests.length > 1) {
|
|
555
|
+
paddingOffset = 0;
|
|
556
|
+
planned = planRequests(state, uniqueSendable.map((index) => questions[index]), target);
|
|
557
|
+
}
|
|
558
|
+
const { requests, oversized } = planned;
|
|
559
|
+
for (const { value, bytes } of oversized) {
|
|
560
|
+
if (value < paddingOffset)
|
|
561
|
+
continue;
|
|
562
|
+
const failure = Result.fail(tooLarge(bytes));
|
|
563
|
+
for (const index of duplicates.get(value - paddingOffset))
|
|
564
|
+
results[index] = failure;
|
|
565
|
+
}
|
|
566
|
+
yield* Effect.forEach(requests, (request) => send(source, state, request, media).pipe(Effect.tap((received) => {
|
|
331
567
|
const answered = [];
|
|
332
568
|
for (const { value, answer } of received) {
|
|
333
|
-
|
|
569
|
+
if (value < paddingOffset)
|
|
570
|
+
continue;
|
|
571
|
+
const targets = duplicates.get(value - paddingOffset);
|
|
572
|
+
const leader = targets[0];
|
|
334
573
|
if (answer === "declined") {
|
|
335
|
-
|
|
574
|
+
const failure = Result.fail(declined);
|
|
575
|
+
for (const index of targets)
|
|
576
|
+
results[index] = failure;
|
|
336
577
|
} else {
|
|
337
|
-
|
|
338
|
-
|
|
578
|
+
const success = Result.succeed(answer);
|
|
579
|
+
for (const index of targets)
|
|
580
|
+
results[index] = success;
|
|
581
|
+
answered.push({ question: questions[leader], print: prints[leader], noul: answer });
|
|
339
582
|
}
|
|
340
583
|
}
|
|
341
|
-
return cache === undefined ? Effect.void : cache.set(state, answered);
|
|
584
|
+
return cache === undefined || answered.length === 0 ? Effect.void : cache.set(state, answered);
|
|
342
585
|
})), { concurrency: "unbounded", discard: true });
|
|
343
586
|
return results;
|
|
344
587
|
});
|
|
@@ -350,9 +593,22 @@ var ItemFields = Schema2.Struct({
|
|
|
350
593
|
id: Schema2.NonEmptyString,
|
|
351
594
|
text: Schema2.NonEmptyString,
|
|
352
595
|
path: Schema2.optional(Schema2.NonEmptyString),
|
|
353
|
-
lines: Schema2.optional(Schema2.Tuple([Schema2.Int, Schema2.Int]).check(Schema2.makeFilter(([start, end]) => 1 <= start && start <= end || "lines must be [start, end] with 1 <= start <= end")))
|
|
596
|
+
lines: Schema2.optional(Schema2.Tuple([Schema2.Int, Schema2.Int]).check(Schema2.makeFilter(([start, end]) => 1 <= start && start <= end || "lines must be [start, end] with 1 <= start <= end"))),
|
|
597
|
+
media: Schema2.optional(Schema2.Array(Schema2.NonEmptyString))
|
|
354
598
|
});
|
|
355
599
|
var formatIssues = SchemaIssue.makeFormatterStandardSchemaV1();
|
|
600
|
+
var mediaProblem = (media) => {
|
|
601
|
+
const untyped = media.find((file) => mediaTypeOf(file) === undefined);
|
|
602
|
+
if (untyped !== undefined)
|
|
603
|
+
return `media ${untyped} is not a file type decision models take (${mediaExtensions.join(", ")})`;
|
|
604
|
+
for (const kind of mediaKinds) {
|
|
605
|
+
const count = media.filter((file) => mediaTypeOf(file).kind === kind).length;
|
|
606
|
+
if (count > maxMediaFiles[kind]) {
|
|
607
|
+
return `media lists ${count} ${mediaNoun[kind]}; an item shows the model at most ${maxMediaFiles[kind]}`;
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
return;
|
|
611
|
+
};
|
|
356
612
|
var parseItem = Effect2.fn("parseItem")(function* (line) {
|
|
357
613
|
const value = yield* Effect2.try({ try: () => JSON.parse(line), catch: () => "not valid JSON" });
|
|
358
614
|
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
@@ -361,12 +617,16 @@ var parseItem = Effect2.fn("parseItem")(function* (line) {
|
|
|
361
617
|
if (unknown.length > 0) {
|
|
362
618
|
return yield* Effect2.fail(`unknown field ${unknown.join(", ")}; put what the model should read in text`);
|
|
363
619
|
}
|
|
364
|
-
|
|
620
|
+
const item = yield* Schema2.decodeUnknownEffect(ItemFields)(value).pipe(Effect2.mapError((error) => formatIssues(error.issue).issues.map(({ path, message }) => `${path?.length ? `${path.join(".")}: ` : ""}${message}`).join("; ")));
|
|
621
|
+
const problem = item.media === undefined ? undefined : mediaProblem(item.media);
|
|
622
|
+
return problem === undefined ? item : yield* Effect2.fail(problem);
|
|
365
623
|
});
|
|
366
624
|
// src/lint.ts
|
|
367
625
|
import { Array as Arr, Effect as Effect4, Result as Result2 } from "effect";
|
|
368
626
|
|
|
369
627
|
// src/select.ts
|
|
628
|
+
import { stat } from "fs/promises";
|
|
629
|
+
import { resolve } from "path";
|
|
370
630
|
import { Effect as Effect3, Runtime as Runtime2, Schema as Schema3 } from "effect";
|
|
371
631
|
class SelectorError extends Schema3.TaggedError()("SelectorError", {
|
|
372
632
|
message: Schema3.String
|
|
@@ -416,6 +676,15 @@ var start = async (select, cwd) => {
|
|
|
416
676
|
running.delete(child.pid);
|
|
417
677
|
}
|
|
418
678
|
};
|
|
679
|
+
var findMedia = (item, cwd) => {
|
|
680
|
+
if (item.media === undefined)
|
|
681
|
+
return Effect3.succeed(item);
|
|
682
|
+
const media = cwd === undefined ? item.media : item.media.map((file) => resolve(cwd, file));
|
|
683
|
+
return Effect3.forEach(media, (file, index) => Effect3.tryPromise({
|
|
684
|
+
try: () => stat(file),
|
|
685
|
+
catch: () => `media ${item.media[index]}: no such file`
|
|
686
|
+
}).pipe(Effect3.filterOrFail((stats) => stats.isFile(), () => `media ${item.media[index]} is not a file`))).pipe(Effect3.as({ ...item, media }));
|
|
687
|
+
};
|
|
419
688
|
var runSelector = Effect3.fn("runSelector")(function* (policy, cwd, runs) {
|
|
420
689
|
const result = yield* Effect3.promise(() => {
|
|
421
690
|
if (runs === undefined)
|
|
@@ -440,14 +709,18 @@ var runSelector = Effect3.fn("runSelector")(function* (policy, cwd, runs) {
|
|
|
440
709
|
}
|
|
441
710
|
const lines = result.stdout.split(`
|
|
442
711
|
`);
|
|
443
|
-
return yield* Effect3.forEach(lines.flatMap((line, index) => line.trim() === "" ? [] : [{ line, number: index + 1 }]), ({ line, number }) => parseItem(line).pipe(Effect3.mapError((reason) => new SelectorError({ message: `${policy.name}: selector output line ${number} is not a valid item: ${reason}` }))));
|
|
712
|
+
return yield* Effect3.forEach(lines.flatMap((line, index) => line.trim() === "" ? [] : [{ line, number: index + 1 }]), ({ line, number }) => parseItem(line).pipe(Effect3.flatMap((item) => findMedia(item, cwd)), Effect3.mapError((reason) => new SelectorError({ message: `${policy.name}: selector output line ${number} is not a valid item: ${reason}` }))));
|
|
444
713
|
});
|
|
445
714
|
|
|
446
715
|
// src/lint.ts
|
|
447
716
|
var reviewFloor = 0.5;
|
|
448
717
|
var defaultThreshold = 0.65;
|
|
449
718
|
var verdictOf = (violates, floor, threshold) => violates <= floor ? "pass" : violates >= threshold ? "violation" : "review";
|
|
450
|
-
var itemQuestion = (policy, item) => ({
|
|
719
|
+
var itemQuestion = (policy, item) => ({
|
|
720
|
+
instructions: item.text,
|
|
721
|
+
criteria: policy.criteria,
|
|
722
|
+
...item.media === undefined || item.media.length === 0 ? {} : { media: item.media }
|
|
723
|
+
});
|
|
451
724
|
var toFinding = (violates, base, policy, threshold) => Result2.match(violates, {
|
|
452
725
|
onSuccess: (violates) => {
|
|
453
726
|
const verdict = verdictOf(violates, reviewFloor, threshold);
|
|
@@ -460,7 +733,8 @@ var itemLocation = (item) => ({
|
|
|
460
733
|
...item.path === undefined ? {} : { path: item.path },
|
|
461
734
|
...item.lines === undefined ? {} : { lines: item.lines }
|
|
462
735
|
});
|
|
463
|
-
var
|
|
736
|
+
var compareStrings = (a, b) => a < b ? -1 : a > b ? 1 : 0;
|
|
737
|
+
var byLocation = (a, b) => compareStrings(a.path ?? "", b.path ?? "") || (a.lines?.[0] ?? 0) - (b.lines?.[0] ?? 0) || compareStrings(a.id, b.id);
|
|
464
738
|
var select = Effect4.fn("select")(function* (policy, cwd, runs) {
|
|
465
739
|
return (yield* runSelector(policy, cwd, runs)).toSorted(byLocation);
|
|
466
740
|
});
|
|
@@ -595,7 +869,35 @@ var loadLabels = Effect6.fn("loadLabels")(function* (policyName, directory = ".l
|
|
|
595
869
|
return yield* parseLabels(file, text);
|
|
596
870
|
});
|
|
597
871
|
var overlaps = (lines, [start, end]) => lines[0] <= end && start <= lines[1];
|
|
872
|
+
var overlapLength = ([a0, a1], [b0, b1]) => Math.max(0, Math.min(a1, b1) - Math.max(a0, b0) + 1);
|
|
598
873
|
var itemLabel = (target, item) => target.label !== "violates" || target.lines === undefined ? target.label : target.lines.some((range) => item.lines === undefined || overlaps(item.lines, range)) ? "violates" : "complies";
|
|
874
|
+
var labelFileItems = (target, items) => {
|
|
875
|
+
if (target.label !== "violates" || target.lines === undefined)
|
|
876
|
+
return items.map(() => target.label);
|
|
877
|
+
const ranges = target.lines;
|
|
878
|
+
return items.map((item) => {
|
|
879
|
+
if (item.lines === undefined)
|
|
880
|
+
return "violates";
|
|
881
|
+
const [start, end] = item.lines;
|
|
882
|
+
const matched = ranges.filter((range) => overlaps(item.lines, range));
|
|
883
|
+
if (matched.length === 0)
|
|
884
|
+
return "complies";
|
|
885
|
+
const onlyNested = matched.every((range) => {
|
|
886
|
+
const rangeLen = range[1] - range[0] + 1;
|
|
887
|
+
const mine = overlapLength(item.lines, range);
|
|
888
|
+
if (mine / rangeLen >= 0.5)
|
|
889
|
+
return false;
|
|
890
|
+
return items.some((other) => {
|
|
891
|
+
if (other === item || other.lines === undefined)
|
|
892
|
+
return false;
|
|
893
|
+
const [os, oe] = other.lines;
|
|
894
|
+
const strictlyEncloses = os <= start && end <= oe && (os < start || oe > end);
|
|
895
|
+
return strictlyEncloses && overlapLength(other.lines, range) > mine;
|
|
896
|
+
});
|
|
897
|
+
});
|
|
898
|
+
return onlyNested ? "complies" : "violates";
|
|
899
|
+
});
|
|
900
|
+
};
|
|
599
901
|
var reachedRanges = (ranges, items) => ranges.filter((range) => items.some((item) => item.lines === undefined || overlaps(item.lines, range))).length;
|
|
600
902
|
var targetScore = (scores) => {
|
|
601
903
|
const answered = scores.filter((score) => score !== null);
|
|
@@ -625,6 +927,55 @@ var separation = (low, high) => {
|
|
|
625
927
|
};
|
|
626
928
|
var scoresOf = (cases, label) => cases.filter((scored) => scored.label === label).map(probability);
|
|
627
929
|
var auroc = (cases) => separation(scoresOf(cases, "complies"), scoresOf(cases, "violates"));
|
|
930
|
+
var auprc = (cases) => {
|
|
931
|
+
const clear = cases.filter((scored) => scored.label !== "ambiguous");
|
|
932
|
+
const totalPos = clear.filter((scored) => scored.label === "violates").length;
|
|
933
|
+
if (totalPos === 0 || clear.length === totalPos)
|
|
934
|
+
return Number.NaN;
|
|
935
|
+
const sorted = clear.map((scored) => ({ score: probability(scored), pos: scored.label === "violates" })).sort((a, b) => b.score - a.score);
|
|
936
|
+
let tp = 0;
|
|
937
|
+
let seen = 0;
|
|
938
|
+
let ap = 0;
|
|
939
|
+
for (let start = 0;start < sorted.length; ) {
|
|
940
|
+
let end = start;
|
|
941
|
+
let groupPos = 0;
|
|
942
|
+
while (end < sorted.length && sorted[end].score === sorted[start].score) {
|
|
943
|
+
if (sorted[end].pos)
|
|
944
|
+
groupPos++;
|
|
945
|
+
end++;
|
|
946
|
+
}
|
|
947
|
+
const groupSize = end - start;
|
|
948
|
+
if (groupPos > 0) {
|
|
949
|
+
let sumPrec = 0;
|
|
950
|
+
for (let k = 1;k <= groupPos; k++) {
|
|
951
|
+
const expectedSeen = seen + k * (groupSize + 1) / (groupPos + 1);
|
|
952
|
+
sumPrec += (tp + k) / expectedSeen;
|
|
953
|
+
}
|
|
954
|
+
ap += sumPrec / totalPos;
|
|
955
|
+
}
|
|
956
|
+
tp += groupPos;
|
|
957
|
+
seen += groupSize;
|
|
958
|
+
start = end;
|
|
959
|
+
}
|
|
960
|
+
return ap;
|
|
961
|
+
};
|
|
962
|
+
var tprAtFpr = (cases, fpr) => {
|
|
963
|
+
const clean = scoresOf(cases, "complies").sort((a, b) => b - a);
|
|
964
|
+
const viol = scoresOf(cases, "violates");
|
|
965
|
+
if (clean.length === 0 || viol.length === 0)
|
|
966
|
+
return Number.NaN;
|
|
967
|
+
const cut = clean[Math.min(clean.length - 1, Math.floor(clean.length * fpr))];
|
|
968
|
+
const above = viol.filter((value) => value > cut).length;
|
|
969
|
+
const tied = viol.filter((value) => value === cut).length;
|
|
970
|
+
const cleanAbove = clean.filter((value) => value > cut).length;
|
|
971
|
+
const cleanTied = clean.filter((value) => value === cut).length;
|
|
972
|
+
const allowedFromTie = cleanTied > 0 ? Math.max(0, Math.min(1, (fpr * clean.length - cleanAbove) / cleanTied)) : 0;
|
|
973
|
+
return (above + tied * allowedFromTie) / viol.length;
|
|
974
|
+
};
|
|
975
|
+
var violationPrecision = (cases, thresholds = cliThresholds) => {
|
|
976
|
+
const clearFlagged = cases.filter((scored) => scored.label !== "ambiguous" && verdict(probability(scored), thresholds) === "violation");
|
|
977
|
+
return clearFlagged.length === 0 ? Number.NaN : clearFlagged.filter((scored) => scored.label === "violates").length / clearFlagged.length;
|
|
978
|
+
};
|
|
628
979
|
var concordance = (cases) => {
|
|
629
980
|
const clean = scoresOf(cases, "complies");
|
|
630
981
|
const ambiguous = scoresOf(cases, "ambiguous");
|
|
@@ -759,7 +1110,10 @@ var scorePolicy = Effect6.fn("scorePolicy")(function* (policy, labels, cwd) {
|
|
|
759
1110
|
}
|
|
760
1111
|
return {
|
|
761
1112
|
targets: ofTarget.map(({ record, items }) => ({ subject: record, label: record.label, group: targetKey(record), score: targetScore(items.map(score)) })),
|
|
762
|
-
items: ofTarget.flatMap(({ record, items }) =>
|
|
1113
|
+
items: ofTarget.flatMap(({ record, items }) => {
|
|
1114
|
+
const itemLabels = labelFileItems(record, items);
|
|
1115
|
+
return items.map((item, index) => ({ subject: item, label: itemLabels[index], group: targetKey(record), score: score(item) }));
|
|
1116
|
+
}),
|
|
763
1117
|
reach
|
|
764
1118
|
};
|
|
765
1119
|
});
|
|
@@ -777,6 +1131,7 @@ export {
|
|
|
777
1131
|
PolicyError,
|
|
778
1132
|
RequestTooLarge,
|
|
779
1133
|
SelectorError,
|
|
1134
|
+
auprc,
|
|
780
1135
|
auroc,
|
|
781
1136
|
bootstrapInterval,
|
|
782
1137
|
brier,
|
|
@@ -800,6 +1155,7 @@ export {
|
|
|
800
1155
|
judge,
|
|
801
1156
|
keyHeader,
|
|
802
1157
|
killRunningSelectors,
|
|
1158
|
+
labelFileItems,
|
|
803
1159
|
lint,
|
|
804
1160
|
loadLabels,
|
|
805
1161
|
loadPolicies,
|
|
@@ -819,9 +1175,11 @@ export {
|
|
|
819
1175
|
sumUsage,
|
|
820
1176
|
targetKey,
|
|
821
1177
|
targetScore,
|
|
1178
|
+
tprAtFpr,
|
|
822
1179
|
tuneThresholds,
|
|
823
1180
|
verdict,
|
|
824
1181
|
verdictF1,
|
|
825
1182
|
verdictOf,
|
|
1183
|
+
violationPrecision,
|
|
826
1184
|
workersAiTarget
|
|
827
1185
|
};
|