@drkmttr/adhere 0.7.3 → 0.9.0

package/src/services/Jev.http.ts

@@ -1,3 +1,4 @@
+import type { RuleId } from "#config.ts";
 import { AdhereConfig } from "#services/AdhereConfig.ts";
 import { Credentials } from "#services/Credentials.ts";
 import {
@@ -10,8 +11,10 @@ import {
   JevBlocked,
   JevOverflow,
   JevUnavailable,
+  type Judged,
   judgeBody,
   type Lines,
+  linterKey,
   locateBody,
   namedPairs,
   needsBlocks,
@@ -19,6 +22,7 @@ import {
   pairProbability,
   requestsOf,
   type Rules,
+  tokensOf,
 } from "#services/Jev.ts";
 import { type Cause, Duration, Effect, Layer, Option, Record, Schedule, Schema } from "effect";
 import { HttpClient, HttpClientError, HttpClientResponse } from "effect/unstable/http";
@@ -36,6 +40,14 @@ const ChoiceAnswers = Schema.Struct({
 
 const refused = (message: string) => JevUnavailable.make({ message });
 
+/** A failed attempt, for the log: the status and Ray ID when Jev's side answered, the cause otherwise. */
+const attemptFailure = (
+  problem: HttpClientError.HttpClientError | RateLimiter.RateLimiterError | Cause.TimeoutError,
+): string =>
+  HttpClientError.isHttpClientError(problem) && problem.reason._tag === "StatusCodeError"
+    ? `HTTP ${problem.reason.response.status}, Ray ID ${problem.reason.response.headers["cf-ray"] ?? "none"}`
+    : problem.message;
+
 /** Jev's error body, on one line and cut short, or nothing when it sent none. */
 const detailOf = (body: string): string => {
   const text = body.trim().replace(/\s+/g, " ");
@@ -68,7 +80,14 @@ const failed = (
   }
   const { response } = problem.reason;
   return Effect.flatMap(
-    Effect.orElseSucceed(response.text, () => ""),
+    Effect.orElseSucceed(response.text, () => "").pipe(
+      Effect.tap((body) =>
+        Effect.logDebug(
+          `Jev answered HTTP ${response.status}, ${response.headers["content-type"] ?? "no content type"}, ${body.length} characters, Ray ID ${response.headers["cf-ray"] ?? "none"}`,
+        ),
+      ),
+      Effect.tap((body) => Effect.logTrace(`Jev's answer: ${body.slice(0, 4000)}`)),
+    ),
     (body): Effect.Effect<never, JevOverflow | JevBlocked | JevUnavailable> =>
       response.status === 400 && isOverflow(body)
         ? Effect.fail(JevOverflow.make())
@@ -124,13 +143,18 @@ export const JevLive = Layer.effect(Jev)(
           );
     const client = paced.pipe(
       HttpClient.transformResponse(Effect.timeout("30 seconds")),
+      // Every failed attempt, including those retried, so a log shows what retrying hid.
+      HttpClient.transformResponse(
+        Effect.tapError((problem) => Effect.logDebug(`attempt failed: ${attemptFailure(problem)}`)),
+      ),
       HttpClient.retryTransient({
         schedule: Schedule.exponential("500 millis"),
         times: 3,
       }),
     );
 
-    const ask = <A>(body: Body, Answers: Schema.Codec<A, unknown, never, never>) =>
+    /** One request. `kind` names the question for the log: judge, locate, blocks, and so on. */
+    const ask = <A>(kind: string, body: Body, Answers: Schema.Codec<A, unknown, never, never>) =>
       Effect.gen(function* () {
         // Read here, not in the layer: a run with nothing pending needs no key.
         const apiKey = yield* credentials.apiKey.pipe(
@@ -140,9 +164,16 @@ export const JevLive = Layer.effect(Jev)(
           HttpClientRequest.post(SYSTEM_ONE),
           body,
         ).pipe(Effect.orDie);
-        const response = yield* client
-          .execute(HttpClientRequest.bearerToken(request, apiKey))
-          .pipe(Effect.catch(failed));
+        const questions = Object.keys(body.questions).length;
+        yield* Effect.logDebug(
+          `${kind}: sending ${questions} ${questions === 1 ? "question" : "questions"}, about ${tokensOf(body)} tokens`,
+        );
+        const [elapsed, response] = yield* Effect.timed(
+          client.execute(HttpClientRequest.bearerToken(request, apiKey)).pipe(Effect.catch(failed)),
+        );
+        yield* Effect.logDebug(
+          `${kind}: HTTP ${response.status} in ${Math.round(Duration.toMillis(elapsed))} ms, Ray ID ${response.headers["cf-ray"] ?? "none"}`,
+        );
         return yield* HttpClientResponse.schemaBodyJson(Answers)(response).pipe(
           Effect.mapError((problem) =>
             refused(`System One response did not decode: ${problem.message}`),
@@ -152,6 +183,7 @@ export const JevLive = Layer.effect(Jev)(
 
     /** Every answer to a body, over as many requests as Jev's context needs. */
     const answersTo = <A>(
+      kind: string,
       body: Body,
       Answers: Schema.Codec<
         { readonly answers: Readonly<Record<string, A>> },
@@ -161,7 +193,7 @@ export const JevLive = Layer.effect(Jev)(
       >,
     ) =>
       Effect.map(
-        Effect.forEach(requestsOf(body), (request) => ask(request, Answers)),
+        Effect.forEach(requestsOf(body), (request) => ask(kind, request, Answers)),
         (responses) =>
           responses.reduce<Record<string, A>>(
             (answers, response) => ({ ...answers, ...response.answers }),
@@ -169,9 +201,28 @@ export const JevLive = Layer.effect(Jev)(
           ),
       );
 
-    const judge = Effect.fn("Jev.judge")(function* (lines: Lines, rules: Rules) {
-      const answers = yield* answersTo(judgeBody(config.model, lines, rules), NoulAnswers);
-      return Record.map(answers, (answer) => answer.noul);
+    const judge = Effect.fn("Jev.judge")(function* (
+      lines: Lines,
+      rules: Rules,
+      sampled: ReadonlyArray<RuleId> = [],
+    ) {
+      const answers = yield* answersTo(
+        "judge",
+        judgeBody(config.model, lines, rules, sampled),
+        NoulAnswers,
+      );
+      /** Each rule's answer, read from the question it rode under. */
+      const answered = (ids: ReadonlyArray<RuleId>, keyOf: (id: RuleId) => string) =>
+        Record.fromEntries(
+          ids.flatMap((id) => {
+            const answer = answers[keyOf(id)];
+            return answer === undefined ? [] : [[id, answer.noul] as const];
+          }),
+        );
+      return {
+        probabilities: answered(Object.keys(rules), (id) => id),
+        linter: answered(sampled, linterKey),
+      } satisfies Judged;
     });
 
     const chosen = (answers: Readonly<Record<string, { choice: string }>>) =>
@@ -179,10 +230,10 @@ export const JevLive = Layer.effect(Jev)(
 
     const locate = Effect.fn("Jev.locate")(function* (lines: Lines, rules: Rules) {
       const blocks = needsBlocks(lines)
-        ? chosen(yield* answersTo(blockBody(config.model, lines, rules), ChoiceAnswers))
+        ? chosen(yield* answersTo("blocks", blockBody(config.model, lines, rules), ChoiceAnswers))
         : undefined;
       return chosen(
-        yield* answersTo(locateBody(config.model, lines, rules, blocks), ChoiceAnswers),
+        yield* answersTo("locate", locateBody(config.model, lines, rules, blocks), ChoiceAnswers),
       );
     });
 
@@ -191,6 +242,7 @@ export const JevLive = Layer.effect(Jev)(
       partners: ReadonlyArray<ReadonlyArray<number>>,
     ) {
       const answers = yield* answersTo(
+        "conflicts",
         conflictBody(config.model, rules, partners),
         ChoiceAnswers,
       ).pipe(refuseOverflow);
@@ -205,7 +257,9 @@ export const JevLive = Layer.effect(Jev)(
         pairs,
         (pair) =>
           Effect.map(
-            ask(contradictBody(config.model, rules, pair), NoulAnswers).pipe(refuseOverflow),
+            ask("contradicts", contradictBody(config.model, rules, pair), NoulAnswers).pipe(
+              refuseOverflow,
+            ),
             ({ answers }) => pairProbability(answers, pair),
           ),
         { concurrency: 8 },

package/src/services/Jev.ts

@@ -24,13 +24,27 @@ export class JevBlocked extends Schema.TaggedError<JevBlocked>()("JevBlocked", {
   ray: Schema.String,
 }) {}
 
+/** What judging a file answers: per rule, whether it breaks it, and the linter check's answers. */
+export interface Judged {
+  /** Per rule, Jev's probability that the file breaks it. */
+  readonly probabilities: Readonly<Record<RuleId, number>>;
+  /** Per sampled rule, Jev's probability that a regular linter could have decided the file against it. */
+  readonly linter: Readonly<Record<RuleId, number>>;
+}
+
 export class Jev extends Context.Service<
   Jev,
   {
+    /**
+     * Each rule's probability that the file breaks it. A rule in `sampled`
+     * also carries the linter check's question, whose answer comes back
+     * under `linter`.
+     */
     readonly judge: (
       lines: Lines,
       rules: Rules,
-    ) => Effect.Effect<Record<RuleId, number>, JevUnavailable | JevOverflow | JevBlocked>;
+      sampled?: ReadonlyArray<RuleId>,
+    ) => Effect.Effect<Judged, JevUnavailable | JevOverflow | JevBlocked>;
     readonly locate: (
       lines: Lines,
       rules: Rules,
@@ -89,7 +103,7 @@ const encoder = new TextEncoder();
 export const tokensOf = (value: unknown): number =>
   Math.ceil(encoder.encode(JSON.stringify(value)).length / 3);
 
-export const snippetOf = (line: string): string => line.trim().slice(0, 120);
+const snippetOf = (line: string): string => line.trim().slice(0, 120);
 
 const isBlank = (line: string): boolean => line.trim().length === 0;
 
@@ -193,10 +207,53 @@ const lineCriteria = (lines: Lines, from: number, to: number): Record<string, st
   return criteria;
 };
 
-export const judgeBody = (model: string, lines: Lines, rules: Rules) => ({
+/**
+ * The linter check's question: whether a regular linter could have decided
+ * `code` against the rule. It carries the judge question's fields, so Jev
+ * sees the rule as it does when judging, with the file in view.
+ */
+export const linterQuestion = (rule: Rule) => {
+  const { instructions } = judgeQuestion(rule);
+  return {
+    type: "noul" as const,
+    instructions: {
+      ...instructions,
+      question:
+        "Should `rule` have been checked by a regular linter? Consider `code`: could a linter or type checker have decided exactly whether it follows `rule`, without judgment?",
+    },
+    criteria: {
+      true: "A regular linter or type checker could decide exactly whether `code` follows `rule`",
+      false:
+        "Deciding whether `code` follows `rule` takes judgment about what the code means or is for",
+    },
+  };
+};
+
+/** The key a rule's linter question rides under, beside its judge question. */
+export const linterKey = (id: RuleId): string => `linter:${id}`;
+
+/** The tokens a file's code leaves in Jev's context for any one question beside it, as `fits` counts them. */
+export const questionRoom = (lines: Lines): number =>
+  QUESTION_CONTEXT - WRAPPER - tokensOf(stateOf(lines));
+
+/** One question per rule, and the linter question beside each rule in `sampled`. */
+export const judgeBody = (
+  model: string,
+  lines: Lines,
+  rules: Rules,
+  sampled: ReadonlyArray<RuleId> = [],
+) => ({
   model,
   state: stateOf(lines),
-  questions: Record.map(rules, (rule) => judgeQuestion(rule)),
+  questions: {
+    ...Record.map(rules, (rule): ReturnType<typeof judgeQuestion> => judgeQuestion(rule)),
+    ...Object.fromEntries(
+      sampled.flatMap((id) => {
+        const rule = rules[id];
+        return rule === undefined ? [] : [[linterKey(id), linterQuestion(rule)] as const];
+      }),
+    ),
+  },
 });
 
 export const locateBody = (
@@ -288,8 +345,11 @@ export const requestsOf = (body: Body): ReadonlyArray<Body> => {
 };
 
 /** The requests judging `rules` over a file takes: the judge body, split to fit, as `judge` sends it. */
-export const judgeRequests = (lines: Lines, rules: Rules): number =>
-  requestsOf(judgeBody("", lines, rules)).length;
+export const judgeRequests = (
+  lines: Lines,
+  rules: Rules,
+  sampled: ReadonlyArray<RuleId> = [],
+): number => requestsOf(judgeBody("", lines, rules, sampled)).length;
 
 /**
  * The requests locating `rules` in a file takes, split to fit as `locate`

package/src/services/SourceWalker.ts

@@ -79,9 +79,16 @@ const scannable = (root: string, paths: ReadonlyArray<string>, self: string, kee
 /** One root's paths, filtered to what the audit reads. */
 const listRoot = Effect.fn("SourceWalker.listRoot")(
   (fs: FileSystem.FileSystem, root: string, self: string, keep: Keep) =>
-    Effect.map(fs.readDirectory(root, { recursive: true }), (paths) =>
-      scannable(root, paths, self, keep),
-    ),
+    Effect.flatMap(fs.readDirectory(root, { recursive: true }), (paths) => {
+      const files = scannable(root, paths, self, keep);
+      // The paths listed against the files kept: node_modules shows as the difference.
+      return Effect.as(
+        Effect.logDebug(
+          `listed ${root}: ${paths.length} paths, ${files.length} source files to read`,
+        ),
+        files,
+      );
+    }),
 );
 
 /** One scannable path, read into lines. */
@@ -89,7 +96,7 @@ const readFile = Effect.fn("SourceWalker.readFile")((fs: FileSystem.FileSystem,
   Effect.map(fs.readFileString(path), (content): ScannedFile => ({
     path,
     lines: content.split("\n"),
-  })),
+  })).pipe(Effect.tap((file) => Effect.logTrace(`read ${path}: ${file.lines.length} lines`))),
 );
 
 /** Read a batch of paths; the inner loop hoisted out of `forEach`. */

package/src/wording.ts

@@ -0,0 +1,90 @@
+import { type Example, examplesOf, type Rule } from "#config.ts";
+import type { RuleEntry } from "#rules.ts";
+
+type Word = Example["word"];
+
+/**
+ * A way a rule's wording differs from the README's rule writing tips, which
+ * the eval measured: the rule shows code of both kinds, and its description
+ * says the words its code is under.
+ */
+export type Straying =
+  /** The rule has code under `word`, and its description does not say it. */
+  | { readonly _tag: "Unsaid"; readonly word: Word }
+  /** The rule has code to write, and no example under `word` of code that breaks it. */
+  | { readonly _tag: "Unshown"; readonly word: Word }
+  /** The description says `word`, which belongs to the other kind of rule. */
+  | { readonly _tag: "Mixed"; readonly word: Word };
+
+/** A rule whose wording differs from the tips, with each way it does. */
+export interface Stray {
+  readonly entry: RuleEntry;
+  readonly straying: ReadonlyArray<Straying>;
+}
+
+export const TIPS_URL = "https://github.com/darkmatter/adhere#rule-writing-tips";
+
+/** Whether a description says a word, whole and in any case: "Must" does, "mustard" does not. */
+const says = (description: string, word: Word): boolean =>
+  new RegExp(`\\b${word.replace(" ", "\\s+")}\\b`, "i").test(description);
+
+/**
+ * How one rule strays from the tips. A rule with only code that breaks it
+ * lacks no example: the README keeps that for a rule with no single correct
+ * form to show.
+ */
+export const strayingOf = (rule: Rule): ReadonlyArray<Straying> => {
+  const { good, bad } = examplesOf(rule);
+  const guideline = good?.word === "should" || bad?.word === "should not";
+  const shown = [good, bad].flatMap((example) => (example === undefined ? [] : [example.word]));
+  const unshown: ReadonlyArray<Word> =
+    good !== undefined && bad === undefined ? [guideline ? "should not" : "never"] : [];
+  // "should" also finds "should not", so a requirement is checked for the one word.
+  const others: ReadonlyArray<Word> = guideline ? ["must", "never"] : ["should"];
+  return [
+    ...shown
+      .filter((word) => !says(rule.description, word))
+      .map((word): Straying => ({ _tag: "Unsaid", word })),
+    ...unshown.map((word): Straying => ({ _tag: "Unshown", word })),
+    ...others
+      .filter((word) => says(rule.description, word))
+      .map((word): Straying => ({ _tag: "Mixed", word })),
+  ];
+};
+
+/** The rules that stray, in the order given. */
+export const straysAmong = (entries: ReadonlyArray<RuleEntry>): ReadonlyArray<Stray> =>
+  entries.flatMap((entry) => {
+    const straying = strayingOf(entry.rule);
+    return straying.length === 0 ? [] : [{ entry, straying }];
+  });
+
+/** One way a rule strays, as a sentence. */
+const sentenceOf = (straying: Straying): string => {
+  switch (straying._tag) {
+    case "Unsaid":
+      return `The description does not say "${straying.word}", though the rule has a ${straying.word} example.`;
+    case "Unshown":
+      return `The rule has no ${straying.word} example. Rules with one example of each kind judge best.`;
+    case "Mixed":
+      return straying.word === "should"
+        ? `The description says "should", a guideline's word, but the rule is a requirement.`
+        : `The description says "${straying.word}", a requirement's word, but the rule is a guideline.`;
+  }
+};
+
+const displayPath = (file: string, root: string): string =>
+  file.startsWith(`${root}/`) ? file.slice(root.length + 1) : file;
+
+/** The wording part of `adhere validate`'s report: each rule that strays, then how. */
+export const formatStrays = (strays: ReadonlyArray<Stray>, root: string): ReadonlyArray<string> =>
+  strays.length === 0
+    ? ["Every rule is worded as the rule writing tips recommend."]
+    : [
+        `${strays.length} ${strays.length === 1 ? "rule is" : "rules are"} not worded as the rule writing tips recommend:`,
+        ...strays.flatMap(({ entry, straying }) => [
+          `  ${entry.id}${entry.file === undefined ? "" : ` (${displayPath(entry.file, root)})`}`,
+          ...straying.map((one) => `    ${sentenceOf(one)}`),
+        ]),
+        `Rule writing tips: ${TIPS_URL}`,
+      ];