@drkmttr/adhere 0.5.0 → 0.6.0

package/README.md

@@ -68,6 +68,7 @@ refusal prints its reason.
 adhere lint                    # audit the working directory
 adhere lint --preset effect    # add a built-in rule set; the config becomes optional
 adhere lint --threshold 0.8    # replace the config's threshold; per-rule thresholds still apply
+adhere lint --yes              # send the requests without asking first
 adhere validate                # load the config and rules, ask Jev whether any contradict
 adhere init [--force]          # scaffold .adhere/config.ts and two example rules
 adhere login                   # save a TypeSafe AI API key for later runs
@@ -79,6 +80,24 @@ Bare `adhere` prints the help, which lists the commands, and
 `adhere <command> --help` lists a command's flags. `adhere --completions <shell>`
 prints a completion script.
 
+### Before and during a run
+
+Before it sends anything, `lint` says on stderr what it found and what judging
+takes:
+
+```text
+200 files and 14 rules: 2800 checks, 1400 cached.
+Judging the other 1400 takes 200 requests to Jev, plus 1 or more for each file with a finding.
+? Send 200 requests to Jev? › (Y/n)
+```
+
+It asks only with a terminal on stdin and stdout, and never when the cache
+answers every check; `--yes` sends without asking. While it runs, a counter on
+stderr shows the files done, the requests sent, and the findings so far, and
+the report goes to stdout. A request Jev refuses stops the run with the file,
+the HTTP status, and Jev's reason. Files finished before it are cached, so
+running again continues from there.
+
 ### What gets read
 
 `adhere lint` reads the `.ts` files under the working directory, except `.d.ts`,
@@ -141,7 +160,7 @@ export default {
   rules: {
     "data/brand-meaningful-primitives": {
       description:
-        "A primitive with semantic meaning, such as an id, email, URL, port, or count, is a branded schema.",
+        "A primitive with semantic meaning, such as an id, email, URL, port, or count, should be a branded schema.",
       reference: `
 const UserId = Schema.String.pipe(Schema.brand("UserId"))
 type UserId = typeof UserId.Type
@@ -254,25 +273,36 @@ beats all of the above for that rule.
 
 ## How a file is judged
 
-1. One Jev request per file, with the file's numbered lines and every rule as
-   the state, and one `noul` (yes/no probability) question per rule: whether
-   the file diverges from the reference or, for a rule with only code to
-   avoid, whether it contains that code. Every question shares the state cost
-   of the request.
+1. One Jev request per file. The state is the file's numbered lines and
+   nothing else. Each rule is one `noul` (yes/no probability) question that
+   carries the rule: whether the file breaks the rule's description, with
+   criteria for each answer. Yes means part of the file breaks the rule, and
+   the code to avoid is an example of it. No means the file follows the rule
+   or has nothing the rule covers, and the reference is an example of it.
+   Since no rule sits in the shared state, a rule's probability depends only
+   on the file and that rule, not on which other rules share the request.
+   Every question shares the state cost of the request.
 2. A second request only when at least one rule's probability is above its
    threshold: one `choice` question per flagged rule over the file's non-blank
    lines, which yields the line to report. A file with more than 255 non-blank
    lines is located in two steps (a block of 20 lines, then a line inside it).
-   Files longer than 5100 lines are skipped and counted in the summary.
+3. Jev reads at most 32k tokens of state and one question together, and 64k
+   tokens in a request. adhere estimates tokens from the JSON it sends, at
+   about three bytes a token. When a file's questions would not fit in one
+   request, they are split across several. A file whose code and longest
+   question would not fit together, or that has more than 5100 lines, is
+   skipped and counted in the summary, and the rest of the run goes on.
 
 ## Cache
 
 Judgments are cached in `.adhere/cache/` (add it to `.gitignore`), one entry
 per file. An entry stores the file's content hash and, per rule, the
-probability, the located line, and a fingerprint of the rule's text and model.
-A changed file re-judges every rule for that file. An edited rule re-judges
-only that rule. A lowered threshold locates cached judgments that are newly
-above it without judging again. Entries depend on content, not on the
+probability, the located line, and a fingerprint of the model and the
+question asked for the rule, which carries the rule's text. A changed file
+re-judges every rule for that file. An edited rule re-judges only that rule,
+and an adhere that asks its questions differently re-judges every rule once.
+A lowered threshold locates cached judgments that are newly above it without
+judging again. Entries depend on content, not on the
 machine, so restoring `.adhere/cache/` between CI runs skips unchanged files.
 
 The API key is read only when a request is about to be sent. A run where

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@drkmttr/adhere",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "private": false,
   "description": "Lints a repository against reference code in .adhere/, judged by Jev.",
   "repository": {
@@ -67,10 +67,10 @@
     }
   },
   "optionalDependencies": {
-    "@drkmttr/adhere-darwin-arm64": "0.5.0",
-    "@drkmttr/adhere-darwin-x64": "0.5.0",
-    "@drkmttr/adhere-linux-arm64": "0.5.0",
-    "@drkmttr/adhere-linux-x64": "0.5.0",
-    "@drkmttr/adhere-win32-x64": "0.5.0"
+    "@drkmttr/adhere-darwin-arm64": "0.6.0",
+    "@drkmttr/adhere-darwin-x64": "0.6.0",
+    "@drkmttr/adhere-linux-arm64": "0.6.0",
+    "@drkmttr/adhere-linux-x64": "0.6.0",
+    "@drkmttr/adhere-win32-x64": "0.6.0"
   }
 }

package/presets/effect/basics/fn-for-named-effects.md

@@ -1,5 +1,5 @@
 ---
-description: A named function that returns an Effect is defined with Effect.fn so the call site is traced.
+description: A named function that returns an Effect should be defined with Effect.fn so the call site is traced.
 ---
 
 ```ts

package/presets/effect/basics/gen-for-sequencing.md

@@ -1,5 +1,5 @@
 ---
-description: Sequential effectful steps are written with Effect.gen and yield*, not nested flatMap or callback chains.
+description: Sequential effectful steps should be written with Effect.gen and yield*, not nested flatMap or callback chains.
 ---
 
 ```ts

package/presets/effect/basics/instrument-with-pipe.md

@@ -1,5 +1,5 @@
 ---
-description: Timeouts, retries, logging, and spans are attached with .pipe, not written into the body of the effect.
+description: Timeouts, retries, logging, and spans should be attached with .pipe, not written into the body of the effect.
 ---
 
 ```ts

package/presets/effect/config/business-logic-depends-on-config-service.md

@@ -1,5 +1,5 @@
 ---
-description: Business logic depends on a config service. Config primitives are read only inside that service's layer.
+description: Business logic depends on a config service. Config primitives should be read only inside that service's layer.
 ---
 
 ```ts

package/presets/effect/config/secrets-are-redacted.md

@@ -1,5 +1,5 @@
 ---
-description: A token, password, or key is read with Config.redacted.
+description: A token, password, or key should be read with Config.redacted.
 ---
 
 ```ts

package/presets/effect/config/validate-with-schema.md

@@ -1,5 +1,5 @@
 ---
-description: A config value with constraints is read with Config.schema, not mapped and checked by hand.
+description: A config value with constraints should be read with Config.schema, not mapped and checked by hand.
 ---
 
 ```ts

package/presets/effect/data/brand-meaningful-primitives.md

@@ -1,5 +1,5 @@
 ---
-description: A primitive with semantic meaning, such as an id, email, URL, port, or count, is a branded schema.
+description: A primitive with semantic meaning, such as an id, email, URL, port, or count, should be a branded schema.
 ---
 
 ```ts

package/presets/effect/data/decode-json-with-schema.md

@@ -1,5 +1,5 @@
 ---
-description: JSON crossing a boundary is decoded with Schema.fromJsonString, not JSON.parse followed by a cast.
+description: JSON crossing a boundary should be decoded with Schema.fromJsonString, not JSON.parse followed by a cast.
 ---
 
 ```ts

package/presets/effect/data/records-are-schema-classes.md

@@ -1,5 +1,5 @@
 ---
-description: Domain records are defined with Schema.Class, not with a plain interface or type alias.
+description: Domain records should be defined with Schema.Class, not with a plain interface or type alias.
 ---
 
 ```ts

package/presets/effect/data/variants-are-tagged-unions.md

@@ -1,5 +1,5 @@
 ---
-description: Structured variants are Schema.TaggedClass in a Schema.Union and are matched with Match.tag and Match.exhaustive.
+description: Structured variants should be Schema.TaggedClass in a Schema.Union and matched with Match.tag and Match.exhaustive.
 ---
 
 ```ts

package/presets/effect/errors/catch-defects-at-boundaries-only.md

@@ -1,5 +1,5 @@
 ---
-description: Defects are caught only at a system boundary for logging or shutdown, never in business logic.
+description: Defects should be caught only at a system boundary for logging or shutdown, never in business logic.
 ---
 
 ```ts

package/presets/effect/errors/defects-for-bugs.md

@@ -1,5 +1,5 @@
 ---
-description: A typed error is for a failure the caller can handle. A bug or invariant violation is a defect.
+description: A typed error should be for a failure the caller can handle. A bug or invariant violation should be a defect.
 ---
 
 ```ts

package/presets/effect/errors/domain-errors-are-tagged.md

@@ -1,5 +1,5 @@
 ---
-description: A domain failure is a Schema.TaggedError with its own tag.
+description: A domain failure should be a Schema.TaggedError with its own tag.
 ---
 
 ```ts

package/presets/effect/errors/wrap-external-errors.md

@@ -1,5 +1,5 @@
 ---
-description: An error from an external library is wrapped in a tagged error with a Schema.Defect field, not passed through raw.
+description: An error from an external library should be wrapped in a tagged error with a Schema.Defect field, not passed through raw.
 ---
 
 ```ts

package/presets/effect/platform/os-access-through-platform-services.md

@@ -1,5 +1,5 @@
 ---
-description: File, path, process, terminal, and HTTP access goes through Effect's platform services (FileSystem, Path, ChildProcess, Terminal, HttpClient, KeyValueStore), acquired with yield*, not through node: builtins, Bun globals, or fetch. The Bun implementations are provided once at the entry point with BunServices.layer.
+description: File, path, process, terminal, and HTTP access goes through Effect's platform services (FileSystem, Path, ChildProcess, Terminal, HttpClient, KeyValueStore), acquired with yield*, not through node: builtins, Bun globals, or fetch. The Bun implementations should be provided once at the entry point with BunServices.layer.
 ---
 
 ```ts

package/presets/effect/services/fresh-layer-per-test.md

@@ -1,5 +1,5 @@
 ---
-description: Each it.effect provides its own layer. it.layer is used only to share an expensive resource across a suite.
+description: Each it.effect provides its own layer. it.layer should be used only to share an expensive resource across a suite.
 ---
 
 ```ts

package/presets/effect/services/memoize-parameterized-layers.md

@@ -1,5 +1,5 @@
 ---
-description: The result of a parameterized layer constructor is stored in a module constant before it is used in more than one place.
+description: The result of a parameterized layer constructor should be stored in a module constant before it is used in more than one place.
 ---
 
 ```ts

package/presets/effect/services/methods-have-no-requirements.md

@@ -1,5 +1,5 @@
 ---
-description: Service method signatures have no requirements. Dependencies are acquired inside the layer, not declared on the method.
+description: Service method signatures have no requirements. Dependencies should be acquired inside the layer, not declared on the method.
 ---
 
 ```ts

package/presets/effect/services/provide-at-entry.md

@@ -1,5 +1,5 @@
 ---
-description: Layers are provided once at the program entry. A module that is not an entry point does not call Effect.provide.
+description: Layers should be provided once at the program entry. A module that is not an entry point does not call Effect.provide.
 ---
 
 ```ts

package/presets/effect/services/test-layers-are-in-memory.md

@@ -1,5 +1,5 @@
 ---
-description: A test implementation of a service is built with Layer.sync or Layer.succeed over in-memory state.
+description: A test implementation of a service should be built with Layer.sync or Layer.succeed over in-memory state.
 ---
 
 ```ts

package/presets/effect/testing/test-clock-for-time.md

@@ -1,5 +1,5 @@
 ---
-description: A test that depends on time uses TestClock. Real sleeps and it.live are used only when real time is required.
+description: A test that depends on time uses TestClock. Real sleeps and it.live should be used only when real time is required.
 ---
 
 ```ts

package/src/cli.ts

@@ -3,7 +3,20 @@ import { AuditCacheLive } from "#services/AuditCache.ts";
 import { Credentials, CredentialsLive, CredentialsUnavailable } from "#services/Credentials.ts";
 import { JevLive } from "#services/Jev.http.ts";
 import { SourceWalkerLive } from "#services/SourceWalker.ts";
-import { render, runAudit } from "#workflows/audit.ts";
+import {
+  type AuditPlan,
+  executeAudit,
+  type FileDone,
+  planAudit,
+  render,
+} from "#workflows/audit.ts";
+import {
+  describePlan,
+  type Progress,
+  progressAfter,
+  progressLine,
+  sendQuestion,
+} from "#workflows/format.ts";
 import type { Overrides } from "#config.ts";
 import { findContradictions, formatContradictions } from "#contradictions.ts";
 import { initProject } from "#init.ts";
@@ -12,6 +25,7 @@ import { globalRuleSet } from "#rules.ts";
 import {
   Console,
   Effect,
+  Exit,
   Layer,
   Option,
   Path,
@@ -54,6 +68,44 @@ const threshold = Flag.float("threshold").pipe(
   ),
 );
 
+const yes = Flag.boolean("yes").pipe(
+  Flag.withAlias("y"),
+  Flag.withDefault(false),
+  Flag.withDescription(
+    "Send the requests without asking first. lint only asks with a terminal on stdin and stdout.",
+  ),
+);
+
+class Cancelled extends Schema.TaggedError<Cancelled>()("Cancelled", {}) {
+  readonly [Runtime.errorReported] = false;
+}
+
+/** Text on stderr, beside the report on stdout. */
+const toStderr = (text: string) =>
+  Effect.sync(() => {
+    process.stderr.write(text);
+  });
+
+/**
+ * A counter on stderr, redrawn in place on a terminal as files finish. On
+ * anything else it stays quiet, so a log gets the plan and the report rather
+ * than a line per file. A failed run keeps its last count on screen.
+ */
+const counterFor = (plan: AuditPlan) => {
+  const live = process.stderr.isTTY === true;
+  let progress: Progress = { files: 0, requests: 0, findings: 0 };
+  const draw = () => (live ? toStderr(`\r\u001b[2K${progressLine(progress, plan)}`) : Effect.void);
+  return {
+    start: Effect.suspend(draw),
+    update: (done: FileDone) =>
+      Effect.suspend(() => {
+        progress = progressAfter(progress, done);
+        return draw();
+      }),
+    finish: (failed: boolean) => (live ? toStderr(failed ? "\n" : "\r\u001b[2K") : Effect.void),
+  };
+};
+
 const force = Flag.boolean("force").pipe(
   Flag.withDefault(false),
   Flag.withDescription("Overwrite existing scaffold files."),
@@ -70,10 +122,28 @@ export const auditLayer = (overrides: Overrides) =>
   ).pipe(Layer.provideMerge(AdhereConfigLive(overrides)));
 
 /** `adhere lint`: the audit. */
-export const lintCommand = Command.make("lint", { preset, threshold }, () =>
+export const lintCommand = Command.make("lint", { preset, threshold, yes }, (input) =>
   Effect.gen(function* () {
-    const result = yield* runAudit;
     const stdio = yield* Stdio.Stdio;
+    const plan = yield* planAudit;
+    yield* toStderr(`${describePlan(plan).join("\n")}\n`);
+    // A prompt draws on stdout, so it needs a terminal at both ends: none when
+    // the report is redirected, or in CI.
+    const interactive = (yield* stdio.stdinIsTerminal) && (yield* stdio.stdoutIsTerminal);
+    if (plan.requests > 0 && interactive && !input.yes) {
+      const send = yield* Prompt.run(
+        Prompt.confirm({ message: sendQuestion(plan), initial: true }),
+      );
+      if (!send) {
+        yield* toStderr("Nothing sent.\n");
+        return yield* Cancelled.make({});
+      }
+    }
+    const counter = counterFor(plan);
+    yield* counter.start;
+    const result = yield* executeAudit(plan, counter.update).pipe(
+      Effect.onExit((exit) => counter.finish(Exit.isFailure(exit))),
+    );
     const color = yield* stdio.stdoutIsTerminal;
     const path = yield* Path.Path;
     // One write: separate Console.log calls have interleaved out of order here.

package/src/services/Jev.http.ts

@@ -2,6 +2,7 @@ import { AdhereConfig } from "#services/AdhereConfig.ts";
 import { Credentials } from "#services/Credentials.ts";
 import {
   blockBody,
+  type Body,
   type ComparedRule,
   conflictBody,
   contradictBody,
@@ -14,10 +15,11 @@ import {
   needsBlocks,
   type Pair,
   pairProbability,
+  requestsOf,
   type Rules,
 } from "#services/Jev.ts";
-import { Effect, Layer, Record, Schedule, Schema } from "effect";
-import { HttpClient, HttpClientResponse } from "effect/unstable/http";
+import { type Cause, Effect, Layer, Record, Schedule, Schema } from "effect";
+import { HttpClient, HttpClientError, HttpClientResponse } from "effect/unstable/http";
 import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest";
 
 const SYSTEM_ONE = "https://api.typesafe.ai/v1/systemone";
@@ -31,6 +33,27 @@ const ChoiceAnswers = Schema.Struct({
 
 const refused = (message: string) => JevUnavailable.make({ message });
 
+/** Jev's error body, on one line and cut short, or nothing when it sent none. */
+const detailOf = (body: string): string => {
+  const text = body.trim().replace(/\s+/g, " ");
+  return text.length === 0 ? "" : `: ${text.slice(0, 300)}`;
+};
+
+/**
+ * A failed request as a refusal: for an answer outside 2xx, its status and
+ * what Jev said, since the status alone does not say why; otherwise the cause.
+ */
+const refusalOf = (problem: HttpClientError.HttpClientError | Cause.TimeoutError) => {
+  if (!HttpClientError.isHttpClientError(problem) || problem.reason._tag !== "StatusCodeError") {
+    return Effect.succeed(refused(`System One request failed: ${problem.message}`));
+  }
+  const { response } = problem.reason;
+  return Effect.map(
+    Effect.orElseSucceed(response.text, () => ""),
+    (body) => refused(`Jev answered HTTP ${response.status}${detailOf(body)}`),
+  );
+};
+
 export const JevLive = Layer.effect(Jev)(
   Effect.gen(function* () {
     const config = yield* AdhereConfig;
@@ -44,7 +67,7 @@ export const JevLive = Layer.effect(Jev)(
       }),
     );
 
-    const ask = <A>(body: unknown, Answers: Schema.Codec<A, unknown, never, never>) =>
+    const ask = <A>(body: Body, Answers: Schema.Codec<A, unknown, never, never>) =>
       Effect.gen(function* () {
         // Read here, not in the layer: a run with nothing pending needs no key.
         const apiKey = yield* credentials.apiKey.pipe(
@@ -56,9 +79,7 @@ export const JevLive = Layer.effect(Jev)(
         ).pipe(Effect.orDie);
         const response = yield* client
           .execute(HttpClientRequest.bearerToken(request, apiKey))
-          .pipe(
-            Effect.mapError((problem) => refused(`System One request failed: ${problem.message}`)),
-          );
+          .pipe(Effect.catch((problem) => Effect.flatMap(refusalOf(problem), Effect.fail)));
         return yield* HttpClientResponse.schemaBodyJson(Answers)(response).pipe(
           Effect.mapError((problem) =>
             refused(`System One response did not decode: ${problem.message}`),
@@ -66,8 +87,27 @@ export const JevLive = Layer.effect(Jev)(
         );
       });
 
+    /** Every answer to a body, over as many requests as Jev's context needs. */
+    const answersTo = <A>(
+      body: Body,
+      Answers: Schema.Codec<
+        { readonly answers: Readonly<Record<string, A>> },
+        unknown,
+        never,
+        never
+      >,
+    ) =>
+      Effect.map(
+        Effect.forEach(requestsOf(body), (request) => ask(request, Answers)),
+        (responses) =>
+          responses.reduce<Record<string, A>>(
+            (answers, response) => ({ ...answers, ...response.answers }),
+            {},
+          ),
+      );
+
     const judge = Effect.fn("Jev.judge")(function* (lines: Lines, rules: Rules) {
-      const { answers } = yield* ask(judgeBody(config.model, lines, rules), NoulAnswers);
+      const answers = yield* answersTo(judgeBody(config.model, lines, rules), NoulAnswers);
       return Record.map(answers, (answer) => answer.noul);
     });
 
@@ -76,17 +116,18 @@ export const JevLive = Layer.effect(Jev)(
 
     const locate = Effect.fn("Jev.locate")(function* (lines: Lines, rules: Rules) {
       const blocks = needsBlocks(lines)
-        ? chosen((yield* ask(blockBody(config.model, lines, rules), ChoiceAnswers)).answers)
+        ? chosen(yield* answersTo(blockBody(config.model, lines, rules), ChoiceAnswers))
         : undefined;
-      const { answers } = yield* ask(locateBody(config.model, lines, rules, blocks), ChoiceAnswers);
-      return chosen(answers);
+      return chosen(
+        yield* answersTo(locateBody(config.model, lines, rules, blocks), ChoiceAnswers),
+      );
     });
 
     const conflicts = Effect.fn("Jev.conflicts")(function* (
       rules: ReadonlyArray<ComparedRule>,
       partners: ReadonlyArray<ReadonlyArray<number>>,
     ) {
-      const { answers } = yield* ask(conflictBody(config.model, rules, partners), ChoiceAnswers);
+      const answers = yield* answersTo(conflictBody(config.model, rules, partners), ChoiceAnswers);
       return namedPairs(answers, partners);
     });
 

package/src/services/Jev.ts

@@ -51,7 +51,26 @@ export type Pair = readonly [number, number];
 /** A `choice` question accepts at most this many criteria. */
 const CHOICE_LIMIT = 255;
 const LINES_PER_BLOCK = 20;
-export const MAX_LINES = CHOICE_LIMIT * LINES_PER_BLOCK;
+const MAX_LINES = CHOICE_LIMIT * LINES_PER_BLOCK;
+
+/**
+ * Jev 1.13's context, from its model card: the state with its longest
+ * question, and the state with every question in the request.
+ */
+const QUESTION_CONTEXT = 32_000;
+const REQUEST_CONTEXT = 64_000;
+/** Room for what Jev wraps around a request, which `tokensOf` does not see. */
+const WRAPPER = 1_000;
+
+const encoder = new TextEncoder();
+
+/**
+ * An estimate, meant to run high, of the tokens Jev reads for a value: one
+ * per three bytes of its JSON. Code runs a little over three bytes a token,
+ * prose about four.
+ */
+export const tokensOf = (value: unknown): number =>
+  Math.ceil(encoder.encode(JSON.stringify(value)).length / 3);
 
 export const snippetOf = (line: string): string => line.trim().slice(0, 120);
 
@@ -60,18 +79,22 @@ const isBlank = (line: string): boolean => line.trim().length === 0;
 export const needsBlocks = (lines: Lines): boolean =>
   lines.filter((line) => !isBlank(line)).length > CHOICE_LIMIT;
 
-/** A rule's text as Jev reads it. Code a rule lacks is left out, not sent empty. */
+/** The file, numbered so a line question can name a line. The only thing the questions share. */
+const stateOf = (lines: Lines) => ({
+  code: lines.map((line, index) => `${index + 1} | ${line}`).join("\n"),
+});
+
+/** One side of a noul: what it means, with the rule's code for that side as an example. */
+const sideOf = (what: string, example: string | undefined) =>
+  example === undefined ? { what } : { what, examples: [example] };
+
+/** A rule's text as a comparison of rules reads it. Code a rule lacks is left out, not sent empty. */
 const ruleState = ({ description, reference, avoid }: Rule) => ({
   description,
   ...(reference === undefined ? {} : { reference }),
   ...(avoid === undefined ? {} : { avoid }),
 });
 
-const stateOf = (lines: Lines, rules: Rules) => ({
-  code: lines.map((line, index) => `${index + 1} | ${line}`).join("\n"),
-  rules: Record.map(rules, (rule) => ruleState(rule)),
-});
-
 /**
  * The compared rules a question mentions, keyed by index: ids alone would
  * collide, since a nested rule can share its id with the rule it shadows.
@@ -87,28 +110,30 @@ const comparedState = (rules: ReadonlyArray<ComparedRule>, mentioned: Iterable<n
   return { rules: state };
 };
 
-const field = (id: RuleId, key: "avoid" | "description" | "reference") =>
-  `state.rules["${id}"].${key}`;
-
 /**
- * A rule with a reference asks whether the file diverges from it, with the
- * code to avoid as an example when there is some. A rule without one asks
- * whether the file contains the code to avoid.
+ * Whether the file breaks the rule. The rule rides in the question, not the
+ * state, so each question reads its own rule and no other. The criteria draw
+ * the line: code to avoid is a yes, and the reference is a no, as is a file
+ * with nothing the rule covers.
  */
-const judgeInstructions = (id: RuleId, rule: Rule): string =>
-  rule.reference === undefined
-    ? `Does state.code contain the pattern shown in ${field(id, "avoid")}, which ${field(id, "description")} rules out? Answer no if nothing in this file resembles it.`
-    : `Does state.code diverge from the pattern shown in ${field(id, "reference")}${
-        rule.avoid === undefined ? "" : `, for example by doing what ${field(id, "avoid")} shows`
-      }, as described by ${field(id, "description")}? Answer no if the pattern does not apply to this file.`;
-
-/** What the offending line does, for the line and block questions. */
-const offense = (id: RuleId, rule: Rule): string =>
-  rule.reference === undefined
-    ? `shows the pattern in ${field(id, "avoid")}`
-    : rule.avoid === undefined
-      ? `diverges from ${field(id, "reference")}`
-      : `diverges from ${field(id, "reference")} or resembles ${field(id, "avoid")}`;
+export const judgeQuestion = (rule: Rule) => ({
+  type: "noul" as const,
+  instructions: { question: "Does `code` break `rule`?", rule: rule.description },
+  criteria: {
+    true: sideOf("Part of `code` breaks `rule`", rule.avoid),
+    false: sideOf(
+      "Every part of `code` that `rule` covers follows it, or `rule` covers no part of `code`",
+      rule.reference,
+    ),
+  },
+});
+
+/** The rule as a line question carries it: the description, and its code on either side. */
+const ruleFields = (rule: Rule) => ({
+  rule: rule.description,
+  ...(rule.reference === undefined ? {} : { follows_rule: rule.reference }),
+  ...(rule.avoid === undefined ? {} : { breaks_rule: rule.avoid }),
+});
 
 const lineCriteria = (lines: Lines, from: number, to: number): Record<string, string> => {
   const criteria: Record<string, string> = {};
@@ -121,11 +146,8 @@ const lineCriteria = (lines: Lines, from: number, to: number): Record<string, st
 
 export const judgeBody = (model: string, lines: Lines, rules: Rules) => ({
   model,
-  state: stateOf(lines, rules),
-  questions: Record.map(rules, (rule, id) => ({
-    type: "noul" as const,
-    instructions: judgeInstructions(id, rule),
-  })),
+  state: stateOf(lines),
+  questions: Record.map(rules, (rule) => judgeQuestion(rule)),
 });
 
 export const locateBody = (
@@ -135,12 +157,15 @@ export const locateBody = (
   blocks?: Readonly<Record<RuleId, number>>,
 ) => ({
   model,
-  state: stateOf(lines, rules),
+  state: stateOf(lines),
   questions: Record.map(rules, (rule, id) => {
     const block = blocks?.[id];
     return {
       type: "choice" as const,
-      instructions: `Which line of state.code most clearly ${offense(id, rule)}?`,
+      instructions: {
+        question: "Which line of `code` most clearly breaks `rule`?",
+        ...ruleFields(rule),
+      },
       criteria:
         block === undefined
           ? lineCriteria(lines, 1, lines.length)
@@ -159,15 +184,74 @@ export const blockBody = (model: string, lines: Lines, rules: Rules) => {
   }
   return {
     model,
-    state: stateOf(lines, rules),
-    questions: Record.map(rules, (rule, id) => ({
+    state: stateOf(lines),
+    questions: Record.map(rules, (rule) => ({
       type: "choice" as const,
-      instructions: `Which block of state.code contains the line that most clearly ${offense(id, rule)}?`,
+      instructions: {
+        question: "Which block of `code` contains the line that most clearly breaks `rule`?",
+        ...ruleFields(rule),
+      },
       criteria,
     })),
   };
 };
 
+/**
+ * Whether Jev can take every request a file may need: its lines within two
+ * line choices, and its code beside the longest question any rule asks.
+ */
+export const fits = (lines: Lines, rules: Rules): boolean => {
+  if (lines.length > MAX_LINES) return false;
+  const located = needsBlocks(lines) ? blockBody("", lines, rules) : locateBody("", lines, rules);
+  const questions = [
+    ...Object.values(judgeBody("", lines, rules).questions),
+    ...Object.values(located.questions),
+  ];
+  const longest = Math.max(0, ...questions.map(tokensOf));
+  return tokensOf(stateOf(lines)) + longest <= QUESTION_CONTEXT - WRAPPER;
+};
+
+export interface Body {
+  readonly model: string;
+  readonly state: unknown;
+  readonly questions: Readonly<Record<string, unknown>>;
+}
+
+/**
+ * A body as requests Jev can take: its questions in order, with a new
+ * request whenever the next question would not fit beside the state and
+ * the questions before it.
+ */
+export const requestsOf = (body: Body): ReadonlyArray<Body> => {
+  const room = REQUEST_CONTEXT - WRAPPER - tokensOf(body.state);
+  const requests: Array<{ readonly questions: Record<string, unknown>; tokens: number }> = [];
+  for (const [id, question] of Object.entries(body.questions)) {
+    const tokens = tokensOf(question);
+    const last = requests.at(-1);
+    if (last !== undefined && last.tokens + tokens <= room) {
+      last.questions[id] = question;
+      last.tokens += tokens;
+    } else {
+      requests.push({ questions: { [id]: question }, tokens });
+    }
+  }
+  return requests.map(({ questions }) => ({ ...body, questions }));
+};
+
+/** The requests judging `rules` over a file takes: the judge body, split to fit, as `judge` sends it. */
+export const judgeRequests = (lines: Lines, rules: Rules): number =>
+  requestsOf(judgeBody("", lines, rules)).length;
+
+/**
+ * The requests locating `rules` in a file takes, split to fit as `locate`
+ * sends them: a block choice first for a long file, then a line choice. For a
+ * long file the line choice is counted before a block is chosen, so it can
+ * run high.
+ */
+export const locateRequests = (lines: Lines, rules: Rules): number =>
+  (needsBlocks(lines) ? requestsOf(blockBody("", lines, rules)).length : 0) +
+  requestsOf(locateBody("", lines, rules)).length;
+
 /** A choice keeps one criterion for "none", which leaves this many for partners. */
 const PARTNERS_PER_QUESTION = CHOICE_LIMIT - 1;
 

package/src/workflows/audit.ts

@@ -2,7 +2,15 @@ import type { Rule, RuleId } from "#config.ts";
 import type { ScannedFile, WalkUnavailable } from "#models/Audit.ts";
 import { AdhereConfig } from "#services/AdhereConfig.ts";
 import { AuditCache, type Judgment, sha256 } from "#services/AuditCache.ts";
-import { Jev, type JevUnavailable, MAX_LINES, snippetOf } from "#services/Jev.ts";
+import {
+  fits,
+  Jev,
+  JevUnavailable,
+  judgeQuestion,
+  judgeRequests,
+  locateRequests,
+  snippetOf,
+} from "#services/Jev.ts";
 import { SourceWalker } from "#services/SourceWalker.ts";
 import { applicableRules } from "#rules.ts";
 import { type Crypto, Effect, Record } from "effect";
@@ -45,47 +53,90 @@ const fileResult = (
 const isEmpty = Record.isEmptyReadonlyRecord;
 
 /**
- * What a judgment depends on besides the file. A rule without code to avoid
- * hashes what every rule hashed before `avoid` existed, so its cached
- * judgments survive the upgrade.
+ * What a judgment depends on besides the file: the model, and the question
+ * asked for the rule, which carries the rule. An edited rule re-judges
+ * itself, and a question asked differently re-judges every rule.
  */
 const fingerprintOf = (model: string, rule: Rule) =>
-  sha256(
-    model +
-      rule.description +
-      (rule.reference ?? "") +
-      (rule.avoid === undefined ? "" : `\u0000avoid\u0000${rule.avoid}`),
+  sha256(`${model}\u0000${JSON.stringify(judgeQuestion(rule))}`);
+
+/** One file before anything is sent: its rules, and which of them the cache answers. */
+export interface FilePlan {
+  readonly file: ScannedFile;
+  /** Too long for Jev: counted, not judged. */
+  readonly skipped: boolean;
+  readonly hash: string;
+  readonly prepared: Readonly<Record<RuleId, PreparedRule>>;
+  /** Cached judgments still valid for the file's content and the rule's text. */
+  readonly kept: Readonly<Record<RuleId, Judgment>>;
+  readonly pending: Readonly<Record<RuleId, PreparedRule>>;
+}
+
+/** A run worked out from the files, the rules, and the cache, before any request. */
+export interface AuditPlan {
+  readonly files: ReadonlyArray<FilePlan>;
+  /** Rules that apply to at least one file that is judged. */
+  readonly rules: number;
+  /** Every pair of a judged file and a rule that applies to it. */
+  readonly checks: number;
+  /** Checks a cached judgment answers. */
+  readonly cached: number;
+  /** Files too long to judge. */
+  readonly skipped: number;
+  /** Requests to Jev that judging the other checks takes. Locating findings adds 1 or more per file. */
+  readonly requests: number;
+}
+
+/** A file finished, for a progress counter: the requests it took and the findings it has. */
+export interface FileDone {
+  readonly requests: number;
+  readonly findings: number;
+}
+
+const rulesOf = (some: Readonly<Record<RuleId, PreparedRule>>) => Record.map(some, (k) => k.rule);
+
+const sizeOf = (some: Readonly<Record<string, unknown>>): number => Object.keys(some).length;
+
+/** A refusal that names the file it stopped at. Files finished before it are already cached. */
+const inFile = (file: ScannedFile) =>
+  Effect.mapError((problem: JevUnavailable) =>
+    JevUnavailable.make({
+      message: `${file.path}: ${problem.message}. Files judged before it are cached, so a rerun continues from there.`,
+    }),
   );
 
-export const runAudit: Effect.Effect<
-  AuditResult,
-  WalkUnavailable | JevUnavailable,
-  AdhereConfig | AuditCache | Crypto.Crypto | Jev | SourceWalker
+export const planAudit: Effect.Effect<
+  AuditPlan,
+  WalkUnavailable,
+  AdhereConfig | AuditCache | Crypto.Crypto | SourceWalker
 > = Effect.gen(function* () {
   const config = yield* AdhereConfig;
-  const jev = yield* Jev;
   const cache = yield* AuditCache;
   const files = yield* (yield* SourceWalker).files;
 
   const configuredRules = (file: string) =>
-    config.scopedRules === undefined
-      ? config.rules
-      : applicableRules(file, config.scopedRules);
+    config.scopedRules === undefined ? config.rules : applicableRules(file, config.scopedRules);
 
-  const rulesOf = (some: Readonly<Record<RuleId, PreparedRule>>) => Record.map(some, (k) => k.rule);
-
-  const auditFile = Effect.fn("audit.file")(function* (file: ScannedFile) {
-    if (file.lines.length > MAX_LINES) return fileResult("skipped", []);
+  const planFile = Effect.fn("audit.plan")(function* (file: ScannedFile) {
+    const rules = configuredRules(file.path);
+    // Too long for Jev's context: skipped and counted, rather than refused mid-run.
+    if (!fits(file.lines, rules)) {
+      return {
+        file,
+        skipped: true,
+        hash: "",
+        prepared: {},
+        kept: {},
+        pending: {},
+      } satisfies FilePlan;
+    }
     const prepared: Record<RuleId, PreparedRule> = yield* Effect.forEach(
-      Object.entries(configuredRules(file.path)),
+      Object.entries(rules),
       ([id, rule]) =>
         Effect.map(
           fingerprintOf(config.model, rule),
           (fingerprint) =>
-            [
-              id,
-              { rule, fingerprint, threshold: rule.threshold ?? config.threshold },
-            ] as const,
+            [id, { rule, fingerprint, threshold: rule.threshold ?? config.threshold }] as const,
         ),
     ).pipe(Effect.map(Record.fromEntries));
     const hash = yield* sha256(file.lines.join("\n"));
@@ -96,64 +147,129 @@ export const runAudit: Effect.Effect<
       (judgment, id) => judgment.fingerprint === prepared[id]?.fingerprint,
     );
     const pending = Record.filter(prepared, (_, id) => kept[id] === undefined);
-
-    const probabilities = isEmpty(pending) ? {} : yield* jev.judge(file.lines, rulesOf(pending));
-    const judged: Record<RuleId, Judgment> = { ...kept };
-    for (const [id, probability] of Object.entries(probabilities)) {
-      const k = pending[id];
-      if (k !== undefined) judged[id] = { fingerprint: k.fingerprint, probability };
-    }
-
-    const flagged = Record.filter(prepared, (k, id) => {
-      const judgment = judged[id];
-      return (
-        judgment !== undefined && judgment.probability > k.threshold && judgment.line === undefined
-      );
-    });
-    const lines = isEmpty(flagged) ? {} : yield* jev.locate(file.lines, rulesOf(flagged));
-    const located = Record.map(judged, (judgment, id) => {
-      const line = lines[id];
-      return line === undefined
-        ? judgment
-        : { ...judgment, line, snippet: snippetOf(file.lines[line - 1] ?? "") };
-    });
-
-    const cached = isEmpty(pending) && isEmpty(flagged);
-    if (!cached) yield* cache.put(file.path, { hash, judgments: located });
-
-    const findings = Object.entries(located)
-      .flatMap(([id, judgment]): ReadonlyArray<Finding> => {
-        const k = prepared[id];
-        return k !== undefined && judgment.line !== undefined && judgment.probability > k.threshold
-          ? [
-              {
-                rule: id,
-                description: k.rule.description,
-                reference: k.rule.reference,
-                avoid: k.rule.avoid,
-                file: file.path,
-                line: judgment.line,
-                snippet: judgment.snippet ?? "",
-                probability: judgment.probability,
-              },
-            ]
-          : [];
-      })
-      .sort((a, b) => a.line - b.line);
-    return fileResult(cached ? "cached" : "judged", findings);
+    return { file, skipped: false, hash, prepared, kept, pending } satisfies FilePlan;
   });
 
   const sorted = [...files].sort((a, b) => a.path.localeCompare(b.path));
-  const results = yield* Effect.forEach(sorted, auditFile, { concurrency: 8 });
-  const count = (status: FileResult["status"]) =>
-    results.filter((result) => result.status === status).length;
+  const planned: ReadonlyArray<FilePlan> = yield* Effect.forEach(sorted, planFile, {
+    concurrency: 8,
+  });
+  const judged = planned.filter((plan) => !plan.skipped);
+  const total = (count: (plan: FilePlan) => number) =>
+    judged.reduce((sum, plan) => sum + count(plan), 0);
   return {
-    files: sorted.length,
-    judged: count("judged"),
-    cached: count("cached"),
-    skipped: count("skipped"),
-    findings: results.flatMap((result) => result.findings),
+    files: planned,
+    rules: new Set(judged.flatMap((plan) => Object.keys(plan.prepared))).size,
+    checks: total((plan) => sizeOf(plan.prepared)),
+    cached: total((plan) => sizeOf(plan.kept)),
+    skipped: planned.length - judged.length,
+    requests: total((plan) =>
+      isEmpty(plan.pending) ? 0 : judgeRequests(plan.file.lines, rulesOf(plan.pending)),
+    ),
   };
 });
 
+/**
+ * Judges what the plan left pending and locates the findings, a file at a
+ * time, telling `progress` as each file finishes.
+ */
+export const executeAudit = (
+  plan: AuditPlan,
+  progress: (done: FileDone) => Effect.Effect<void> = () => Effect.void,
+): Effect.Effect<AuditResult, JevUnavailable, AuditCache | Jev> =>
+  Effect.gen(function* () {
+    const jev = yield* Jev;
+    const cache = yield* AuditCache;
+
+    const auditFile = Effect.fn("audit.file")(function* ({
+      file,
+      skipped,
+      hash,
+      prepared,
+      kept,
+      pending,
+    }: FilePlan) {
+      if (skipped) {
+        yield* progress({ requests: 0, findings: 0 });
+        return fileResult("skipped", []);
+      }
+      const probabilities = isEmpty(pending)
+        ? {}
+        : yield* jev.judge(file.lines, rulesOf(pending)).pipe(inFile(file));
+      const judged: Record<RuleId, Judgment> = { ...kept };
+      for (const [id, probability] of Object.entries(probabilities)) {
+        const k = pending[id];
+        if (k !== undefined) judged[id] = { fingerprint: k.fingerprint, probability };
+      }
+
+      const flagged = Record.filter(prepared, (k, id) => {
+        const judgment = judged[id];
+        return (
+          judgment !== undefined &&
+          judgment.probability > k.threshold &&
+          judgment.line === undefined
+        );
+      });
+      const lines = isEmpty(flagged)
+        ? {}
+        : yield* jev.locate(file.lines, rulesOf(flagged)).pipe(inFile(file));
+      const located = Record.map(judged, (judgment, id) => {
+        const line = lines[id];
+        return line === undefined
+          ? judgment
+          : { ...judgment, line, snippet: snippetOf(file.lines[line - 1] ?? "") };
+      });
+
+      const cached = isEmpty(pending) && isEmpty(flagged);
+      if (!cached) yield* cache.put(file.path, { hash, judgments: located });
+
+      const findings = Object.entries(located)
+        .flatMap(([id, judgment]): ReadonlyArray<Finding> => {
+          const k = prepared[id];
+          return k !== undefined &&
+            judgment.line !== undefined &&
+            judgment.probability > k.threshold
+            ? [
+                {
+                  rule: id,
+                  description: k.rule.description,
+                  reference: k.rule.reference,
+                  avoid: k.rule.avoid,
+                  file: file.path,
+                  line: judgment.line,
+                  snippet: judgment.snippet ?? "",
+                  probability: judgment.probability,
+                },
+              ]
+            : [];
+        })
+        .sort((a, b) => a.line - b.line);
+      yield* progress({
+        requests:
+          (isEmpty(pending) ? 0 : judgeRequests(file.lines, rulesOf(pending))) +
+          (isEmpty(flagged) ? 0 : locateRequests(file.lines, rulesOf(flagged))),
+        findings: findings.length,
+      });
+      return fileResult(cached ? "cached" : "judged", findings);
+    });
+
+    const results = yield* Effect.forEach(plan.files, auditFile, { concurrency: 8 });
+    const count = (status: FileResult["status"]) =>
+      results.filter((result) => result.status === status).length;
+    return {
+      files: plan.files.length,
+      judged: count("judged"),
+      cached: count("cached"),
+      skipped: count("skipped"),
+      findings: results.flatMap((result) => result.findings),
+    };
+  });
+
+/** Plans the run and executes it, without asking and without a counter. */
+export const runAudit: Effect.Effect<
+  AuditResult,
+  WalkUnavailable | JevUnavailable,
+  AdhereConfig | AuditCache | Crypto.Crypto | Jev | SourceWalker
+> = Effect.flatMap(planAudit, (plan) => executeAudit(plan));
+
 export { type RenderOptions, render } from "#workflows/format.ts";

package/src/workflows/format.ts

@@ -1,5 +1,5 @@
 import { type Kind, tokenize } from "#highlight.ts";
-import type { AuditResult, Finding } from "#workflows/audit.ts";
+import type { AuditPlan, AuditResult, FileDone, Finding } from "#workflows/audit.ts";
 
 /**
  * The report as data: lines of spans, each span a part of the frame or a
@@ -143,3 +143,44 @@ export const render = (result: AuditResult, options: RenderOptions = {}): Readon
     ...result.findings.flatMap((finding) => [...frame(finding, options.root), []]),
     ...summary(result),
   ].map((line) => serialize(line, options.color === true));
+
+/**
+ * What a run will do, before it asks: the files and rules found, the checks
+ * between them and how many the cache answers, then what judging the rest
+ * takes.
+ */
+export const describePlan = (plan: AuditPlan): ReadonlyArray<string> => {
+  const cached = plan.cached > 0 ? `, ${plan.cached} cached` : "";
+  const skipped =
+    plan.skipped > 0 ? `, ${counted(plan.skipped, "file", "files")} too long to judge` : "";
+  const found = `${counted(plan.files.length, "file", "files")} and ${counted(plan.rules, "rule", "rules")}: ${counted(plan.checks, "check", "checks")}${cached}${skipped}.`;
+  if (plan.requests === 0) {
+    return [found, plan.checks === 0 ? "Nothing to judge." : "The cache answers every check."];
+  }
+  const rest = plan.cached > 0 ? `the other ${plan.checks - plan.cached}` : "them";
+  return [
+    found,
+    `Judging ${rest} takes ${counted(plan.requests, "request", "requests")} to Jev, plus 1 or more for each file with a finding.`,
+  ];
+};
+
+/** The question before a run sends anything. */
+export const sendQuestion = (plan: AuditPlan): string =>
+  `Send ${counted(plan.requests, "request", "requests")} to Jev?`;
+
+/** A run's progress so far: every file it finished, with the requests and findings they added up to. */
+export interface Progress {
+  readonly files: number;
+  readonly requests: number;
+  readonly findings: number;
+}
+
+export const progressAfter = (progress: Progress, done: FileDone): Progress => ({
+  files: progress.files + 1,
+  requests: progress.requests + done.requests,
+  findings: progress.findings + done.findings,
+});
+
+/** The counter a run redraws as files finish. */
+export const progressLine = (progress: Progress, plan: AuditPlan): string =>
+  `${progress.files}/${plan.files.length} files, ${counted(progress.requests, "request", "requests")} sent, ${counted(progress.findings, "finding", "findings")}`;