@drkmttr/adhere 0.7.1 → 0.8.0

package/src/services/AuditCache.ts

@@ -1,21 +1,60 @@
 import { CACHE_DIRECTORY } from "#config.ts";
-import { Context, Crypto, Effect, Layer, Option, Path, Schema } from "effect";
-import * as KeyValueStore from "effect/unstable/persistence/KeyValueStore";
+import { Context, Crypto, Effect, FileSystem, Layer, Option, Path, Record, Schema } from "effect";
 
+/** A rule's judgment as a run holds it: Jev's answer, with the fingerprint of the rule's text. */
 export const Judgment = Schema.Struct({
   fingerprint: Schema.String,
   probability: Schema.Finite,
   line: Schema.optionalKey(Schema.Finite),
-  snippet: Schema.optionalKey(Schema.String),
 });
 export interface Judgment extends Schema.Schema.Type<typeof Judgment> {}
 
+/**
+ * An entry as adhere wrote it before the cache was keyed by content: one per
+ * file, under the hash of the file's absolute path. Read only to carry its
+ * judgments over. Entries written before the report read its excerpt from the
+ * file also hold the line's text, as `snippet`; decoding drops it.
+ */
 export const CacheEntry = Schema.Struct({
   hash: Schema.String,
   judgments: Schema.Record(Schema.String, Judgment),
 });
 export interface CacheEntry extends Schema.Schema.Type<typeof CacheEntry> {}
 
+/** Jev's answer about some code for one rule text: the probability, and the line once located. */
+export const Answer = Schema.Struct({
+  probability: Schema.Finite,
+  line: Schema.optionalKey(Schema.Finite),
+});
+export interface Answer extends Schema.Schema.Type<typeof Answer> {}
+
+/** Answers about one file's content, each under the fingerprint of the rule text it answers. */
+export type Answers = Readonly<Record<string, Answer>>;
+
+const Entry = Schema.Struct({ answers: Schema.Record(Schema.String, Answer) });
+const decodeEntry = Schema.decodeUnknownOption(Schema.fromJsonString(Entry));
+const decodeOldEntry = Schema.decodeUnknownOption(Schema.fromJsonString(CacheEntry));
+
+/**
+ * A rule's linter check so far: per file lint asked it on, Jev's probability
+ * that a regular linter could have decided the file against the rule.
+ */
+export const Tally = Schema.Struct({
+  files: Schema.Record(Schema.String, Schema.Finite),
+});
+export interface Tally extends Schema.Schema.Type<typeof Tally> {}
+const decodeTally = Schema.decodeUnknownOption(Schema.fromJsonString(Tally));
+
+/** What a run that read every file still needs from the cache. */
+export interface Live {
+  /** The hash of each file's content. */
+  readonly hashes: ReadonlySet<string>;
+  /** The fingerprint of each rule text asked about a file. */
+  readonly fingerprints: ReadonlySet<string>;
+  /** The key of each rule's tally. */
+  readonly tallies: ReadonlySet<string>;
+}
+
 const hexOf = (bytes: Uint8Array): string =>
   Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
 
@@ -25,38 +64,248 @@ export const sha256 = Effect.fn("AuditCache.sha256")(function* (text: string) {
   return hexOf(digest);
 });
 
+/** Judgments by rule, as the answers the cache keeps: by the fingerprint of each rule's text. */
+export const answersOf = (judgments: Readonly<Record<string, Judgment>>): Answers =>
+  Object.fromEntries(
+    Object.values(judgments).map(({ fingerprint, ...answer }) => [fingerprint, answer]),
+  );
+
 export class AuditCache extends Context.Service<
   AuditCache,
   {
-    readonly get: (path: string) => Effect.Effect<CacheEntry | undefined>;
-    readonly put: (path: string, entry: CacheEntry) => Effect.Effect<void>;
+    /**
+     * Answers about content with this hash. `path` is where that content is
+     * now, to carry over an entry written before the cache was keyed by content.
+     */
+    readonly get: (hash: string, path: string) => Effect.Effect<Answers>;
+    /** Keeps answers about content with this hash, beside the ones already kept. */
+    readonly put: (hash: string, answers: Answers) => Effect.Effect<void>;
+    /** A rule's tally, under a key that changes with the rule's text. */
+    readonly tally: (key: string) => Effect.Effect<Tally | undefined>;
+    readonly putTally: (key: string, tally: Tally) => Effect.Effect<void>;
+    /**
+     * Deletes what `live` does not need, and folds the rest into one file per
+     * content and per tally. Only after a run that read every file: it cannot
+     * tell what the files it did not read need. Succeeds with the files deleted.
+     */
+    readonly prune: (live: Live) => Effect.Effect<number>;
   }
 >()("@drkmttr/adhere/services/AuditCache") {}
 
-const openStore = Effect.fn("AuditCache.openStore")(function* (directory: string) {
-  const context = yield* Layer.build(KeyValueStore.layerFileSystem(directory)).pipe(
-    Effect.catchCause(() => Layer.build(KeyValueStore.layerMemory)),
-  );
-  return Context.get(context, KeyValueStore.KeyValueStore);
-});
+/** Keys in order, so the same content always writes the same text. */
+const sorted = <A>(record: Readonly<Record<string, A>>): Record<string, A> =>
+  Object.fromEntries(Object.entries(record).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
+
+const textOf = (value: unknown): string => `${JSON.stringify(value, null, 2)}\n`;
+
+/** One answer per rule text: one that has its line over one that has not, otherwise the first. */
+const unionOf = (entries: ReadonlyArray<Answers>): Answers => {
+  const union: Record<string, Answer> = {};
+  for (const answers of entries) {
+    for (const [fingerprint, answer] of Object.entries(answers)) {
+      const held = union[fingerprint];
+      if (held === undefined || (held.line === undefined && answer.line !== undefined)) {
+        union[fingerprint] = answer;
+      }
+    }
+  }
+  return union;
+};
 
+/** One answer per file, the first. */
+const tallyUnionOf = (tallies: ReadonlyArray<Tally>): Tally => {
+  const files: Record<string, number> = {};
+  for (const tally of tallies) {
+    for (const [file, probability] of Object.entries(tally.files)) files[file] ??= probability;
+  }
+  return { files };
+};
+
+/** A group's file: the group, then the hash of the file's text. */
+const NAME = /^([^.]+)\.[0-9a-f]{16}\.json$/;
+/** A file from the layout before content keys: the hash of a path or of a tally's key. */
+const OLD_NAME = /^[0-9a-f]{64}$/;
+
+/**
+ * `.adhere/cache/`, which is meant to be committed. `files/` holds answers by
+ * the content they are about, and `tallies/` the linter check's tallies, with
+ * paths from the working directory. A file is written once and never
+ * rewritten: it is named for its group, a content hash or a tally key, and for
+ * the hash of its own text. Two branches that add to a group add different
+ * files, or the same one, so git merges them without a conflict. Reading a
+ * group takes the union of its files, and pruning folds them back into one.
+ *
+ * A cache must never fail a run: whatever cannot be read counts as missing,
+ * and whatever cannot be written is left unwritten.
+ */
 export const AuditCacheLive = Layer.effect(AuditCache)(
   Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
     const path = yield* Path.Path;
-    const store = yield* openStore(path.join(path.resolve(), CACHE_DIRECTORY));
-    const entries = KeyValueStore.toSchemaStore(store, CacheEntry);
     const crypto = yield* Crypto.Crypto;
-    // Hashed keys: a raw path would name the cache file like a source file.
-    const keyOf = (filePath: string) =>
-      sha256(filePath).pipe(Effect.provideService(Crypto.Crypto, crypto));
-    return AuditCache.of({
-      get: (filePath) =>
-        Effect.flatMap(keyOf(filePath), (key) => entries.get(key)).pipe(
-          Effect.map(Option.getOrUndefined),
-          Effect.orElseSucceed(() => undefined),
+    const root = path.resolve();
+    const directory = path.join(root, CACHE_DIRECTORY);
+    const hash = (text: string) => sha256(text).pipe(Effect.provideService(Crypto.Crypto, crypto));
+
+    /** Each group's file names in one folder, listed once and kept current as files come and go. */
+    const indexOf = (folder: string) =>
+      Effect.cached(
+        fs.readDirectory(path.join(directory, folder)).pipe(
+          Effect.orElseSucceed((): ReadonlyArray<string> => []),
+          Effect.map((names) => {
+            const groups = new Map<string, Set<string>>();
+            for (const name of names) {
+              const group = NAME.exec(name)?.[1];
+              if (group !== undefined)
+                groups.set(group, (groups.get(group) ?? new Set()).add(name));
+            }
+            return groups;
+          }),
         ),
-      put: (filePath, entry) =>
-        Effect.flatMap(keyOf(filePath), (key) => entries.set(key, entry)).pipe(Effect.ignore),
+      );
+    const indexes = { files: yield* indexOf("files"), tallies: yield* indexOf("tallies") };
+    type Folder = keyof typeof indexes;
+
+    const read = <A>(decode: (text: string) => Option.Option<A>, file: string) =>
+      fs.readFileString(file).pipe(
+        Effect.map((text) => Option.getOrUndefined(decode(text))),
+        Effect.orElseSucceed(() => undefined),
+      );
+
+    /** A group's files that decode, in name order. */
+    const readGroup = Effect.fn("AuditCache.readGroup")(function* <A>(
+      folder: Folder,
+      group: string,
+      decode: (text: string) => Option.Option<A>,
+    ) {
+      const names = [...((yield* indexes[folder]).get(group) ?? [])].sort();
+      const decoded = yield* Effect.forEach(names, (name) =>
+        read(decode, path.join(directory, folder, name)),
+      );
+      return decoded.filter((value): value is A => value !== undefined);
+    });
+
+    const nameOf = (group: string, text: string) =>
+      Effect.map(hash(text), (digest) => `${group}.${digest.slice(0, 16)}.json`);
+
+    /** Adds `value` to a group as a new file, unless the group has that file already. */
+    const add = Effect.fn("AuditCache.add")(function* (
+      folder: Folder,
+      group: string,
+      value: unknown,
+    ) {
+      const text = textOf(value);
+      const name = yield* nameOf(group, text);
+      const index = yield* indexes[folder];
+      const names = index.get(group) ?? new Set<string>();
+      if (names.has(name)) return;
+      const written = yield* fs
+        .makeDirectory(path.join(directory, folder), { recursive: true })
+        .pipe(
+          Effect.andThen(fs.writeFileString(path.join(directory, folder, name), text)),
+          Effect.as(true),
+          Effect.orElseSucceed(() => false),
+        );
+      if (written) index.set(group, names.add(name));
+    });
+
+    const remove = (file: string) =>
+      fs.remove(file).pipe(
+        Effect.as(1),
+        Effect.orElseSucceed(() => 0),
+      );
+
+    /** Replaces a group's files with one holding `value`, or with none. Succeeds with the files deleted. */
+    const fold = Effect.fn("AuditCache.fold")(function* (
+      folder: Folder,
+      group: string,
+      value: unknown,
+    ) {
+      const index = yield* indexes[folder];
+      const names = [...(index.get(group) ?? [])];
+      const keep = value === undefined ? undefined : yield* nameOf(group, textOf(value));
+      if (value !== undefined) yield* add(folder, group, value);
+      let deleted = 0;
+      for (const name of names) {
+        if (name !== keep) deleted += yield* remove(path.join(directory, folder, name));
+      }
+      if (keep === undefined) index.delete(group);
+      else index.set(group, new Set([keep]));
+      return deleted;
+    });
+
+    const relativeOf = (tally: Tally): Tally => ({
+      files: sorted(
+        Record.mapKeys(tally.files, (file) =>
+          path.isAbsolute(file) ? path.relative(root, file) : file,
+        ),
+      ),
+    });
+
+    const get = Effect.fn("AuditCache.get")(function* (contentHash: string, filePath: string) {
+      const entries = yield* readGroup("files", contentHash, decodeEntry);
+      if (entries.length > 0) return unionOf(entries.map((entry) => entry.answers));
+      // Before content keys, a file's entry sat under the hash of its absolute path.
+      const old = yield* read(decodeOldEntry, path.join(directory, yield* hash(filePath)));
+      if (old === undefined || old.hash !== contentHash) return {};
+      const answers = answersOf(old.judgments);
+      yield* add("files", contentHash, { answers: sorted(answers) });
+      return answers;
+    });
+
+    const tally = Effect.fn("AuditCache.tally")(function* (key: string) {
+      const tallies = yield* readGroup("tallies", key, decodeTally);
+      if (tallies.length > 0) {
+        const { files } = tallyUnionOf(tallies);
+        return { files: Record.mapKeys(files, (file) => path.resolve(root, file)) };
+      }
+      // Before, a tally sat under the hash of its key, with absolute paths.
+      const old = yield* read(
+        decodeTally,
+        path.join(directory, yield* hash(`\u0000tally\u0000${key}`)),
+      );
+      if (old !== undefined) yield* add("tallies", key, relativeOf(old));
+      return old;
+    });
+
+    const prune = Effect.fn("AuditCache.prune")(function* (live: Live) {
+      // Tallies still in the layout before content keys move first, so deleting what is left of it loses nothing.
+      yield* Effect.forEach(live.tallies, tally, { discard: true });
+      let deleted = 0;
+      // Folding replaces or removes the group being visited, which a Map's iteration allows.
+      for (const group of (yield* indexes.files).keys()) {
+        const entries = live.hashes.has(group) ? yield* readGroup("files", group, decodeEntry) : [];
+        const answers = Record.filter(
+          unionOf(entries.map((entry) => entry.answers)),
+          (_, fingerprint) => live.fingerprints.has(fingerprint),
+        );
+        deleted += yield* fold(
+          "files",
+          group,
+          Object.keys(answers).length === 0 ? undefined : { answers: sorted(answers) },
+        );
+      }
+      for (const key of (yield* indexes.tallies).keys()) {
+        const tallies = live.tallies.has(key) ? yield* readGroup("tallies", key, decodeTally) : [];
+        deleted += yield* fold(
+          "tallies",
+          key,
+          tallies.length === 0 ? undefined : relativeOf(tallyUnionOf(tallies)),
+        );
+      }
+      const old = yield* fs.readDirectory(directory).pipe(Effect.orElseSucceed(() => []));
+      for (const name of old) {
+        if (OLD_NAME.test(name)) deleted += yield* remove(path.join(directory, name));
+      }
+      return deleted;
+    });
+
+    return AuditCache.of({
+      get,
+      put: (contentHash, answers) => add("files", contentHash, { answers: sorted(answers) }),
+      tally,
+      putTally: (key, value) => add("tallies", key, relativeOf(value)),
+      prune,
     });
   }),
 );

package/src/services/Jev.http.ts

@@ -1,3 +1,4 @@
+import type { RuleId } from "#config.ts";
 import { AdhereConfig } from "#services/AdhereConfig.ts";
 import { Credentials } from "#services/Credentials.ts";
 import {
@@ -7,10 +8,13 @@ import {
   conflictBody,
   contradictBody,
   Jev,
+  JevBlocked,
   JevOverflow,
   JevUnavailable,
+  type Judged,
   judgeBody,
   type Lines,
+  linterKey,
   locateBody,
   namedPairs,
   needsBlocks,
@@ -18,6 +22,7 @@ import {
   pairProbability,
   requestsOf,
   type Rules,
+  tokensOf,
 } from "#services/Jev.ts";
 import { type Cause, Duration, Effect, Layer, Option, Record, Schedule, Schema } from "effect";
 import { HttpClient, HttpClientError, HttpClientResponse } from "effect/unstable/http";
@@ -35,6 +40,14 @@ const ChoiceAnswers = Schema.Struct({
 
 const refused = (message: string) => JevUnavailable.make({ message });
 
+/** A failed attempt, for the log: the status and Ray ID when Jev's side answered, the cause otherwise. */
+const attemptFailure = (
+  problem: HttpClientError.HttpClientError | RateLimiter.RateLimiterError | Cause.TimeoutError,
+): string =>
+  HttpClientError.isHttpClientError(problem) && problem.reason._tag === "StatusCodeError"
+    ? `HTTP ${problem.reason.response.status}, Ray ID ${problem.reason.response.headers["cf-ray"] ?? "none"}`
+    : problem.message;
+
 /** Jev's error body, on one line and cut short, or nothing when it sent none. */
 const detailOf = (body: string): string => {
   const text = body.trim().replace(/\s+/g, " ");
@@ -48,6 +61,11 @@ const Overflowed = Schema.fromJsonString(
 const isOverflow = (body: string): boolean =>
   Option.isSome(Schema.decodeUnknownOption(Overflowed)(body));
 
+/** A firewall's block page: HTML where Jev answers JSON, even for its own errors. */
+const isBlockPage = (response: HttpClientResponse.HttpClientResponse, body: string): boolean =>
+  (response.headers["content-type"] ?? "").includes("text/html") ||
+  body.trimStart().startsWith("<");
+
 /**
  * A request that failed. A 400 saying the request is over Jev's context is a
  * `JevOverflow`, which skips the file. Any other answer outside 2xx refuses
@@ -62,20 +80,41 @@ const failed = (
   }
   const { response } = problem.reason;
   return Effect.flatMap(
-    Effect.orElseSucceed(response.text, () => ""),
-    (body): Effect.Effect<never, JevOverflow | JevUnavailable> =>
+    Effect.orElseSucceed(response.text, () => "").pipe(
+      Effect.tap((body) =>
+        Effect.logDebug(
+          `Jev answered HTTP ${response.status}, ${response.headers["content-type"] ?? "no content type"}, ${body.length} characters, Ray ID ${response.headers["cf-ray"] ?? "none"}`,
+        ),
+      ),
+      Effect.tap((body) => Effect.logTrace(`Jev's answer: ${body.slice(0, 4000)}`)),
+    ),
+    (body): Effect.Effect<never, JevOverflow | JevBlocked | JevUnavailable> =>
       response.status === 400 && isOverflow(body)
         ? Effect.fail(JevOverflow.make())
-        : Effect.fail(refused(`Jev answered HTTP ${response.status}${detailOf(body)}`)),
+        : response.status === 403 && isBlockPage(response, body)
+          ? Effect.fail(JevBlocked.make({ ray: response.headers["cf-ray"] ?? "none given" }))
+          : Effect.fail(refused(`Jev answered HTTP ${response.status}${detailOf(body)}`)),
   );
 };
 
-/** Comparing rules has no file to skip, so a request over Jev's context refuses instead. */
+/**
+ * Comparing rules has no file to skip, so a request over Jev's context, or one
+ * the firewall blocks, refuses instead.
+ */
 const refuseOverflow = <A>(
-  self: Effect.Effect<A, JevOverflow | JevUnavailable>,
+  self: Effect.Effect<A, JevOverflow | JevBlocked | JevUnavailable>,
 ): Effect.Effect<A, JevUnavailable> =>
-  Effect.catchTag(self, "JevOverflow", () =>
-    Effect.fail(refused("Comparing the rules took a request over Jev's context")),
+  self.pipe(
+    Effect.catchTag("JevOverflow", () =>
+      Effect.fail(refused("Comparing the rules took a request over Jev's context")),
+    ),
+    Effect.catchTag("JevBlocked", ({ ray }) =>
+      Effect.fail(
+        refused(
+          `The firewall in front of Jev's API blocked comparing the rules (Cloudflare Ray ID ${ray})`,
+        ),
+      ),
+    ),
   );
 
 export const JevLive = Layer.effect(Jev)(
@@ -104,13 +143,18 @@ export const JevLive = Layer.effect(Jev)(
           );
     const client = paced.pipe(
       HttpClient.transformResponse(Effect.timeout("30 seconds")),
+      // Every failed attempt, including those retried, so a log shows what retrying hid.
+      HttpClient.transformResponse(
+        Effect.tapError((problem) => Effect.logDebug(`attempt failed: ${attemptFailure(problem)}`)),
+      ),
       HttpClient.retryTransient({
         schedule: Schedule.exponential("500 millis"),
         times: 3,
       }),
     );
 
-    const ask = <A>(body: Body, Answers: Schema.Codec<A, unknown, never, never>) =>
+    /** One request. `kind` names the question for the log: judge, locate, blocks, and so on. */
+    const ask = <A>(kind: string, body: Body, Answers: Schema.Codec<A, unknown, never, never>) =>
       Effect.gen(function* () {
         // Read here, not in the layer: a run with nothing pending needs no key.
         const apiKey = yield* credentials.apiKey.pipe(
@@ -120,9 +164,16 @@ export const JevLive = Layer.effect(Jev)(
           HttpClientRequest.post(SYSTEM_ONE),
           body,
         ).pipe(Effect.orDie);
-        const response = yield* client
-          .execute(HttpClientRequest.bearerToken(request, apiKey))
-          .pipe(Effect.catch(failed));
+        const questions = Object.keys(body.questions).length;
+        yield* Effect.logDebug(
+          `${kind}: sending ${questions} ${questions === 1 ? "question" : "questions"}, about ${tokensOf(body)} tokens`,
+        );
+        const [elapsed, response] = yield* Effect.timed(
+          client.execute(HttpClientRequest.bearerToken(request, apiKey)).pipe(Effect.catch(failed)),
+        );
+        yield* Effect.logDebug(
+          `${kind}: HTTP ${response.status} in ${Math.round(Duration.toMillis(elapsed))} ms, Ray ID ${response.headers["cf-ray"] ?? "none"}`,
+        );
         return yield* HttpClientResponse.schemaBodyJson(Answers)(response).pipe(
           Effect.mapError((problem) =>
             refused(`System One response did not decode: ${problem.message}`),
@@ -132,6 +183,7 @@ export const JevLive = Layer.effect(Jev)(
 
     /** Every answer to a body, over as many requests as Jev's context needs. */
     const answersTo = <A>(
+      kind: string,
       body: Body,
       Answers: Schema.Codec<
         { readonly answers: Readonly<Record<string, A>> },
@@ -141,7 +193,7 @@ export const JevLive = Layer.effect(Jev)(
       >,
     ) =>
       Effect.map(
-        Effect.forEach(requestsOf(body), (request) => ask(request, Answers)),
+        Effect.forEach(requestsOf(body), (request) => ask(kind, request, Answers)),
         (responses) =>
           responses.reduce<Record<string, A>>(
             (answers, response) => ({ ...answers, ...response.answers }),
@@ -149,9 +201,28 @@ export const JevLive = Layer.effect(Jev)(
           ),
       );
 
-    const judge = Effect.fn("Jev.judge")(function* (lines: Lines, rules: Rules) {
-      const answers = yield* answersTo(judgeBody(config.model, lines, rules), NoulAnswers);
-      return Record.map(answers, (answer) => answer.noul);
+    const judge = Effect.fn("Jev.judge")(function* (
+      lines: Lines,
+      rules: Rules,
+      sampled: ReadonlyArray<RuleId> = [],
+    ) {
+      const answers = yield* answersTo(
+        "judge",
+        judgeBody(config.model, lines, rules, sampled),
+        NoulAnswers,
+      );
+      /** Each rule's answer, read from the question it rode under. */
+      const answered = (ids: ReadonlyArray<RuleId>, keyOf: (id: RuleId) => string) =>
+        Record.fromEntries(
+          ids.flatMap((id) => {
+            const answer = answers[keyOf(id)];
+            return answer === undefined ? [] : [[id, answer.noul] as const];
+          }),
+        );
+      return {
+        probabilities: answered(Object.keys(rules), (id) => id),
+        linter: answered(sampled, linterKey),
+      } satisfies Judged;
     });
 
     const chosen = (answers: Readonly<Record<string, { choice: string }>>) =>
@@ -159,10 +230,10 @@ export const JevLive = Layer.effect(Jev)(
 
     const locate = Effect.fn("Jev.locate")(function* (lines: Lines, rules: Rules) {
       const blocks = needsBlocks(lines)
-        ? chosen(yield* answersTo(blockBody(config.model, lines, rules), ChoiceAnswers))
+        ? chosen(yield* answersTo("blocks", blockBody(config.model, lines, rules), ChoiceAnswers))
         : undefined;
       return chosen(
-        yield* answersTo(locateBody(config.model, lines, rules, blocks), ChoiceAnswers),
+        yield* answersTo("locate", locateBody(config.model, lines, rules, blocks), ChoiceAnswers),
       );
     });
 
@@ -171,6 +242,7 @@ export const JevLive = Layer.effect(Jev)(
       partners: ReadonlyArray<ReadonlyArray<number>>,
     ) {
       const answers = yield* answersTo(
+        "conflicts",
         conflictBody(config.model, rules, partners),
         ChoiceAnswers,
       ).pipe(refuseOverflow);
@@ -185,7 +257,9 @@ export const JevLive = Layer.effect(Jev)(
         pairs,
         (pair) =>
           Effect.map(
-            ask(contradictBody(config.model, rules, pair), NoulAnswers).pipe(refuseOverflow),
+            ask("contradicts", contradictBody(config.model, rules, pair), NoulAnswers).pipe(
+              refuseOverflow,
+            ),
             ({ answers }) => pairProbability(answers, pair),
           ),
         { concurrency: 8 },

package/src/services/Jev.ts

@@ -14,17 +14,41 @@ export class JevUnavailable extends Schema.TaggedError<JevUnavailable>()("JevUna
  */
 export class JevOverflow extends Schema.TaggedError<JevOverflow>()("JevOverflow", {}) {}
 
+/**
+ * A request the firewall in front of Jev's API refused: a 403 with an HTML
+ * block page where Jev answers JSON. It reads some code as an attack, so the
+ * same request is refused every time while the rest of a run goes through.
+ * `ray` is Cloudflare's Ray ID, which TypeSafe AI can look the block up by.
+ */
+export class JevBlocked extends Schema.TaggedError<JevBlocked>()("JevBlocked", {
+  ray: Schema.String,
+}) {}
+
+/** What judging a file answers: per rule, whether it breaks it, and the linter check's answers. */
+export interface Judged {
+  /** Per rule, Jev's probability that the file breaks it. */
+  readonly probabilities: Readonly<Record<RuleId, number>>;
+  /** Per sampled rule, Jev's probability that a regular linter could have decided the file against it. */
+  readonly linter: Readonly<Record<RuleId, number>>;
+}
+
 export class Jev extends Context.Service<
   Jev,
   {
+    /**
+     * Each rule's probability that the file breaks it. A rule in `sampled`
+     * also carries the linter check's question, whose answer comes back
+     * under `linter`.
+     */
     readonly judge: (
       lines: Lines,
       rules: Rules,
-    ) => Effect.Effect<Record<RuleId, number>, JevUnavailable | JevOverflow>;
+      sampled?: ReadonlyArray<RuleId>,
+    ) => Effect.Effect<Judged, JevUnavailable | JevOverflow | JevBlocked>;
     readonly locate: (
       lines: Lines,
       rules: Rules,
-    ) => Effect.Effect<Record<RuleId, number>, JevUnavailable | JevOverflow>;
+    ) => Effect.Effect<Record<RuleId, number>, JevUnavailable | JevOverflow | JevBlocked>;
     /**
      * Per rule, the partner Jev names as impossible to follow in the same code,
      * as `[rule, partner]` indexes into `rules`. A rule Jev names none for is
@@ -79,7 +103,7 @@ const encoder = new TextEncoder();
 export const tokensOf = (value: unknown): number =>
   Math.ceil(encoder.encode(JSON.stringify(value)).length / 3);
 
-export const snippetOf = (line: string): string => line.trim().slice(0, 120);
+const snippetOf = (line: string): string => line.trim().slice(0, 120);
 
 const isBlank = (line: string): boolean => line.trim().length === 0;
 
@@ -183,10 +207,53 @@ const lineCriteria = (lines: Lines, from: number, to: number): Record<string, st
   return criteria;
 };
 
-export const judgeBody = (model: string, lines: Lines, rules: Rules) => ({
+/**
+ * The linter check's question: whether a regular linter could have decided
+ * `code` against the rule. It carries the judge question's fields, so Jev
+ * sees the rule as it does when judging, with the file in view.
+ */
+export const linterQuestion = (rule: Rule) => {
+  const { instructions } = judgeQuestion(rule);
+  return {
+    type: "noul" as const,
+    instructions: {
+      ...instructions,
+      question:
+        "Should `rule` have been checked by a regular linter? Consider `code`: could a linter or type checker have decided exactly whether it follows `rule`, without judgment?",
+    },
+    criteria: {
+      true: "A regular linter or type checker could decide exactly whether `code` follows `rule`",
+      false:
+        "Deciding whether `code` follows `rule` takes judgment about what the code means or is for",
+    },
+  };
+};
+
+/** The key a rule's linter question rides under, beside its judge question. */
+export const linterKey = (id: RuleId): string => `linter:${id}`;
+
+/** The tokens a file's code leaves in Jev's context for any one question beside it, as `fits` counts them. */
+export const questionRoom = (lines: Lines): number =>
+  QUESTION_CONTEXT - WRAPPER - tokensOf(stateOf(lines));
+
+/** One question per rule, and the linter question beside each rule in `sampled`. */
+export const judgeBody = (
+  model: string,
+  lines: Lines,
+  rules: Rules,
+  sampled: ReadonlyArray<RuleId> = [],
+) => ({
   model,
   state: stateOf(lines),
-  questions: Record.map(rules, (rule) => judgeQuestion(rule)),
+  questions: {
+    ...Record.map(rules, (rule): ReturnType<typeof judgeQuestion> => judgeQuestion(rule)),
+    ...Object.fromEntries(
+      sampled.flatMap((id) => {
+        const rule = rules[id];
+        return rule === undefined ? [] : [[linterKey(id), linterQuestion(rule)] as const];
+      }),
+    ),
+  },
 });
 
 export const locateBody = (
@@ -278,8 +345,11 @@ export const requestsOf = (body: Body): ReadonlyArray<Body> => {
 };
 
 /** The requests judging `rules` over a file takes: the judge body, split to fit, as `judge` sends it. */
-export const judgeRequests = (lines: Lines, rules: Rules): number =>
-  requestsOf(judgeBody("", lines, rules)).length;
+export const judgeRequests = (
+  lines: Lines,
+  rules: Rules,
+  sampled: ReadonlyArray<RuleId> = [],
+): number => requestsOf(judgeBody("", lines, rules, sampled)).length;
 
 /**
  * The requests locating `rules` in a file takes, split to fit as `locate`