@drkmttr/adhere 0.10.0 → 0.11.0

package/src/workflows/audit.ts

@@ -1,6 +1,6 @@
 import { isNote, withoutComments } from "#comments.ts";
 import { type Examples, examplesOf, type Level, type Rule, type RuleId } from "#config.ts";
-import { type Excerpt, excerptOf } from "#excerpt.ts";
+import { type Excerpt, type Range } from "#excerpt.ts";
 import type { ScannedFile, WalkUnavailable } from "#models/Audit.ts";
 import { AdhereConfig } from "#services/AdhereConfig.ts";
 import { AuditCache, answersOf, type Judgment, sha256, type Tally } from "#services/AuditCache.ts";
@@ -13,7 +13,9 @@ import {
   judgeLoad,
   judgeRequests,
   linterQuestion,
+  type Located,
   locateRequests,
+  matcherQuestions,
   questionRoom,
   tokensOf,
 } from "#services/Jev.ts";
@@ -32,9 +34,18 @@ export interface Finding {
   readonly description: string;
   readonly examples: Examples;
   readonly file: string;
-  readonly line: number;
-  /** The code around `line` the report shows. */
+  /** The section of the file Jev points at, which a finding is in. */
+  readonly section: Range;
+  /** The lines in the section the code that breaks the rule starts and ends on, when Jev named them. */
+  readonly lines?: Range;
+  /** The section's code as written, comments and all, without blank lines at either end. */
   readonly excerpt: Excerpt;
+  /**
+   * Jev's probability that the file does not show enough to check the rule,
+   * when its probability that it does is below the sufficiency threshold:
+   * the report warns, and says how to add the context.
+   */
+  readonly insufficiency?: number;
   readonly probability: number;
   /** An error fails the run; a warning does not, unless `--deny-warnings`. */
   readonly level: Level;
@@ -90,13 +101,51 @@ const fileResult = (
 
 const isEmpty = Record.isEmptyReadonlyRecord;
 
+/** How Jev reads a file, which a judgment depends on: a new layout re-judges every rule. */
+const LAYOUT = "sections";
+
 /**
- * What a judgment depends on besides the file: the model, and the question
- * asked for the rule, which carries the rule. An edited rule re-judges
- * itself, and a question asked differently re-judges every rule.
+ * What a judgment depends on besides the file: the model, how the file is
+ * laid out, and the questions asked for the rule, which carry the rule: its
+ * judge question, and its matchers' when it has any. An edited rule or
+ * matcher re-judges the rule, and a question asked differently re-judges
+ * every rule.
  */
-const fingerprintOf = (model: string, rule: Rule) =>
-  sha256(`${model}\u0000${JSON.stringify(judgeQuestion(rule))}`);
+const fingerprintOf = (model: string, rule: Rule) => {
+  const matchers = matcherQuestions("", rule).map(([, question]) => question);
+  const scope = matchers.length === 0 ? "" : `\u0000${JSON.stringify(matchers)}`;
+  return sha256(`${model}\u0000${LAYOUT}\u0000${JSON.stringify(judgeQuestion(rule))}${scope}`);
+};
+
+/** A section's lines as written, without the blank lines at either end. */
+const excerptOf = (lines: ReadonlyArray<string>, { first, last }: Range): Excerpt => {
+  let start = first;
+  let end = last;
+  while (start < end && (lines[start - 1] ?? "").trim() === "") start += 1;
+  while (end > start && (lines[end - 1] ?? "").trim() === "") end -= 1;
+  return { start, lines: lines.slice(start - 1, end) };
+};
+
+/** A matcher's score counts as a yes above this. */
+const MATCHER_THRESHOLD = 0.5;
+
+/** Whether a judgment's matchers let it stand: a yes to every `appliesTo`, and to no `excludeIf`. */
+const inScope = (judgment: Judgment): boolean =>
+  (judgment.appliesTo ?? []).every((score) => score > MATCHER_THRESHOLD) &&
+  !(judgment.excludeIf ?? []).some((score) => score > MATCHER_THRESHOLD);
+
+/** The matchers that dropped a judgment, with their scores, as the debug log names them. */
+const scopeMisses = (rule: Rule, judgment: Judgment): string =>
+  [
+    ...(rule.appliesTo ?? []).flatMap((matcher, index) => {
+      const score = judgment.appliesTo?.[index] ?? 0;
+      return score > MATCHER_THRESHOLD ? [] : [`appliesTo "${matcher}" (${score.toFixed(2)})`];
+    }),
+    ...(rule.excludeIf ?? []).flatMap((matcher, index) => {
+      const score = judgment.excludeIf?.[index] ?? 0;
+      return score > MATCHER_THRESHOLD ? [`excludeIf "${matcher}" (${score.toFixed(2)})`] : [];
+    }),
+  ].join(", ");
 
 /** One file before anything is sent: its rules, and which of them the cache answers. */
 export interface FilePlan {
@@ -143,6 +192,8 @@ export interface AuditPlan {
   readonly tokens: number;
   /** The model's price per million input tokens, in dollars, when adhere knows it. */
   readonly price?: number;
+  /** Below this, a finding's sufficiency adds a warning to it. */
+  readonly sufficiencyThreshold: number;
 }
 
 /** A file finished, for a progress counter: the requests it took and the findings it has. */
@@ -408,6 +459,7 @@ export const planAudit = (
       requests: loads.reduce((sum, load) => sum + load.requests, 0),
       tokens: loads.reduce((sum, load) => sum + load.tokens, 0),
       ...(price === undefined ? {} : { price }),
+      sufficiencyThreshold: config.sufficiencyThreshold,
     };
   });
 
@@ -457,14 +509,25 @@ export const executeAudit = (
       deferred,
       sampled,
     }: FilePlan) {
-      const { probabilities, linter }: Judged = isEmpty(pending)
+      const { probabilities, linter, matchers }: Judged = isEmpty(pending)
         ? { probabilities: {}, linter: {} }
         : yield* jev.judge(file.lines, rulesOf(pending), Object.keys(sampled)).pipe(inFile(file));
       if (!isEmpty(linter)) yield* record(file.path, sampled, linter);
       const judged: Record<RuleId, Judgment> = { ...kept };
       for (const [id, probability] of Object.entries(probabilities)) {
         const k = pending[id];
-        if (k !== undefined) judged[id] = { fingerprint: k.fingerprint, probability };
+        if (k !== undefined) {
+          judged[id] = { fingerprint: k.fingerprint, probability, ...matchers?.[id] };
+        }
+      }
+      // Above its threshold, but its matchers put the code out of the rule's scope.
+      for (const [id, judgment] of Object.entries(judged)) {
+        const k = prepared[id];
+        if (k !== undefined && judgment.probability > k.threshold && !inScope(judgment)) {
+          yield* Effect.logDebug(
+            `${file.path}: ${id} (${judgment.probability.toFixed(2)}) dropped by ${scopeMisses(k.rule, judgment)}`,
+          );
+        }
       }
 
       const flagged = Record.filter(prepared, (k, id) => {
@@ -472,29 +535,30 @@ export const executeAudit = (
         return (
           judgment !== undefined &&
           judgment.probability > k.threshold &&
-          judgment.line === undefined
+          inScope(judgment) &&
+          (judgment.section === undefined || judgment.sufficiency === undefined)
         );
       });
-      // A blocked line question still leaves the judgments worth caching: a rerun
-      // asks only for the lines again, not for every rule.
+      // A blocked locate question still leaves the judgments worth caching: a rerun
+      // asks only for the sections again, not for every rule.
       const {
-        lines,
+        places,
         blocked,
       }: {
-        readonly lines: Readonly<Record<RuleId, number>>;
+        readonly places: Readonly<Record<RuleId, Located>>;
         readonly blocked: Blocked | undefined;
       } = isEmpty(flagged)
-        ? { lines: {}, blocked: undefined }
+        ? { places: {}, blocked: undefined }
         : yield* jev.locate(file.lines, rulesOf(flagged)).pipe(
-            Effect.map((lines) => ({ lines, blocked: undefined })),
+            Effect.map((places) => ({ places, blocked: undefined })),
             Effect.catchTag("JevBlocked", ({ ray }) =>
-              Effect.succeed({ lines: {}, blocked: { file: file.path, ray } }),
+              Effect.succeed({ places: {}, blocked: { file: file.path, ray } }),
             ),
             inFile(file),
           );
       const located = Record.map(judged, (judgment, id) => {
-        const line = lines[id];
-        return line === undefined ? judgment : { ...judgment, line };
+        const place = places[id];
+        return place === undefined ? judgment : { ...judgment, ...place };
       });
 
       const cached = isEmpty(pending) && isEmpty(flagged);
@@ -504,8 +568,9 @@ export const executeAudit = (
         .flatMap(([id, judgment]): ReadonlyArray<Finding> => {
           const k = prepared[id];
           return k !== undefined &&
-            judgment.line !== undefined &&
-            judgment.probability > k.threshold
+            judgment.section !== undefined &&
+            judgment.probability > k.threshold &&
+            inScope(judgment)
             ? [
                 {
                   rule: id,
@@ -513,15 +578,20 @@ export const executeAudit = (
                   description: k.rule.description,
                   examples: examplesOf(k.rule),
                   file: file.path,
-                  line: judgment.line,
-                  excerpt: excerptOf(original, judgment.line),
+                  section: judgment.section,
+                  ...(judgment.lines === undefined ? {} : { lines: judgment.lines }),
+                  excerpt: excerptOf(original, judgment.section),
+                  ...(judgment.sufficiency !== undefined &&
+                  judgment.sufficiency < plan.sufficiencyThreshold
+                    ? { insufficiency: 1 - judgment.sufficiency }
+                    : {}),
                   probability: judgment.probability,
                   level: k.rule.level ?? "error",
                 },
               ]
             : [];
         })
-        .sort((a, b) => a.line - b.line);
+        .sort((a, b) => (a.lines ?? a.section).first - (b.lines ?? b.section).first);
       return {
         result:
           blocked === undefined
@@ -583,7 +653,7 @@ export const executeAudit = (
           !suppresses(
             plan.suppressions,
             shownId({ id: finding.rule, preset: finding.preset }),
-            finding.line,
+            finding.lines ?? finding.section,
           ),
       );
       if (shown.length < result.findings.length) {

package/src/workflows/format.ts

@@ -102,36 +102,74 @@ const header = (finding: Finding): Line => [
 ];
 
 /**
- * The path, then the finding's excerpt with each line numbered in the gutter,
- * and an underline under the offending line's code.
+ * The path at the lines Jev points at, or the section's first line when it
+ * named none, then the section with each line numbered in the gutter. One
+ * line Jev points at is underlined under its code; several are marked by a
+ * bar in a column of their own beside the code, as `vp lint` marks a span,
+ * so each line of code keeps one line of the report.
  */
 const excerpt = (finding: Finding, root: string | undefined): ReadonlyArray<Line> => {
   const { start, lines } = finding.excerpt;
   const width = String(start + lines.length - 1).length;
   const gutter = span(" ".repeat(width + 2));
-  const offending = lines[finding.line - start] ?? "";
-  // The line's own indentation, tabs and all, lines the underline up under its code.
-  const indent = offending.slice(0, offending.length - offending.trimStart().length);
-  const underline = "─".repeat(Math.max(1, offending.trim().length));
-  const location = `:${finding.line}:${indent.length + 1}]`;
+  const indentOf = (line: string) => line.slice(0, line.length - line.trimStart().length);
+  const pointed = finding.lines;
+  const at = pointed?.first ?? start;
+  const location = `:${at}:${indentOf(lines[at - start] ?? "").length + 1}]`;
+  const spanning = pointed !== undefined && pointed.last > pointed.first;
+  const inside = (line: number) =>
+    pointed !== undefined && pointed.first <= line && line <= pointed.last;
   return [
     [gutter, span("╭─["), span(displayPath(finding.file, root), "path"), span(location)],
     ...highlighted(lines.join("\n")).flatMap((code, index): ReadonlyArray<Line> => {
       const number = String(start + index);
+      const marker = spanning ? [inside(start + index) ? span("┃ ", "underline") : span("  ")] : [];
       const numbered = [
         span(" ".repeat(width - number.length + 1)),
         span(number, "lineNumber"),
         span(" │ "),
+        ...marker,
         ...code,
       ];
-      return start + index === finding.line
-        ? [numbered, [gutter, span("· "), span(indent), span(underline, "underline")]]
+      const text = lines[index] ?? "";
+      // The line's own indentation, tabs and all, lines the underline up under its code.
+      return !spanning && inside(start + index) && text.trim() !== ""
+        ? [
+            numbered,
+            [
+              gutter,
+              span("· "),
+              span(indentOf(text)),
+              span("─".repeat(text.trim().length), "underline"),
+            ],
+          ]
         : [numbered];
     }),
     [gutter, span("╰────")],
   ];
 };
 
+/**
+ * The warning on a finding whose file may not show enough to check its rule,
+ * with Jev's probability that it does not, and help: how to add what the
+ * code relies on as a note Jev reads, in the form the README asks for.
+ */
+const insufficient = (finding: Finding): ReadonlyArray<Line> =>
+  finding.insufficiency === undefined
+    ? []
+    : [
+        [
+          span("  warning: ", "warning"),
+          span(
+            `this file may not show enough to check this rule (Jev: ${finding.insufficiency.toFixed(2)} that it does not)`,
+          ),
+        ],
+        ...hanging(span("     help: ", "label"), [
+          [span("add what the code relies on outside this file as a note Jev reads:")],
+          highlighted("// @adhere <the fact>, and how you know it")[0] ?? [],
+        ]),
+      ];
+
 /**
  * The code to write as the hint. A rule with only code not to write shows
  * that instead, labeled with its word: `never`, or `should not`.
@@ -145,12 +183,14 @@ const hint = (finding: Finding): ReadonlyArray<Line> => {
 
 /**
  * One diagnostic, in the frame `vp lint` prints on a terminal: a header with
- * the rule in red, the code around the offending line, a pink underline under
- * it, and the code to write as the hint.
+ * the rule in red, the section of the file Jev points at with the lines it
+ * names in pink, a warning when the file may not show enough to decide, and
+ * the code to write as the hint.
  */
 const frame = (finding: Finding, root: string | undefined): ReadonlyArray<Line> => [
   header(finding),
   ...excerpt(finding, root),
+  ...insufficient(finding),
   ...hint(finding),
 ];
 

package/presets/alchemy/data/parameterized-sql.md

@@ -1,17 +0,0 @@
----
-description: SQL must pass values as parameters, through tagged-template interpolation, sql.in, sql.insert, or prepare().bind(), never by string concatenation or a template string handed to an unsafe or raw API.
----
-
-## Must
-
-```ts
-const user = yield* sql`SELECT * FROM users WHERE email = ${email}`;
-const row = yield* Effect.promise(() => d1.prepare("SELECT * FROM users WHERE id = ?").bind(id).first());
-```
-
-## Never
-
-```ts
-const user = yield* sql.unsafe(`SELECT * FROM users WHERE email = '${email}'`);
-const row = yield* Effect.promise(() => d1.prepare("SELECT * FROM users WHERE id = " + id).first());
-```

package/presets/alchemy/secrets/never-logged-returned-or-output.md

@@ -1,19 +0,0 @@
----
-description: A secret, credential, connection string, token, signed URL, or private key must never be logged, returned in an HTTP response, or returned as a stack output; code must expose only names, presence, or non-secret metadata.
----
-
-## Must
-
-```ts
-if (url.pathname === "/secret") {
-  return yield* HttpServerResponse.json({ name: "API_TOKEN", set: Redacted.value(token).length > 0 });
-}
-return { serial: certificate.serial, expires: certificate.notAfter }; // stack output
-```
-
-## Never
-
-```ts
-yield* Effect.log(`token=${Redacted.value(token)}`);
-return { databaseUrl: connection.databaseUrl, tlsKey: certificate.privateKey }; // stack output
-```

package/presets/effect/basics/external-calls-are-resilient.md

@@ -1,21 +0,0 @@
----
-description: A call over the network, such as an HTTP request, a database query, or a third-party API, should carry a timeout and a retry schedule, and should not go out bare. A call through a client that already applies both, such as an HttpClient built with a timeout and retryTransient or an SDK with its own retries, has them. Calls through the platform FileSystem and Path services are local and are not in scope.
----
-
-## Should
-
-```ts
-const retryPolicy = Schedule.exponential("100 millis").pipe(Schedule.both(Schedule.recurs(3)));
-
-const resilientCall = HttpClient.get("https://api.example.com/users").pipe(
-  Effect.timeout("2 seconds"),
-  Effect.retry(retryPolicy),
-  Effect.timeout("10 seconds"),
-);
-```
-
-## Should not
-
-```ts
-const response = yield* HttpClient.get(`https://api.example.com/users/${id}`);
-```