@drkmttr/adhere 0.15.0 → 0.18.0

package/src/services/Jev.http.ts

@@ -1,10 +1,20 @@
-import type { AppendState, JevState, JudgedFile, RuleId } from "#config.ts";
+import type { AppendState, JevState, JudgedFile, ReadState, RuleId } from "#config.ts";
 import { sectionsOf } from "#excerpt.ts";
+import { jev } from "#providers/jev.ts";
+import {
+  Answered,
+  type Provider,
+  type Question,
+  RequestFailed,
+  type State,
+} from "#providers/provider.ts";
+import { questionsOf, type SystemOneQuestion } from "#providers/systemOne.ts";
 import { AdhereConfig } from "#services/AdhereConfig.ts";
 import { Credentials } from "#services/Credentials.ts";
 import {
   type Body,
   type ComparedRule,
+  describeState,
   edgeKey,
   conflictBody,
   contradictBody,
@@ -27,62 +37,50 @@ import {
   type Rules,
   sufficiencyKey,
   tokensOf,
+  logRequestPayload,
 } from "#services/Jev.ts";
 import {
-  type Cause,
   Duration,
   Effect,
   Layer,
-  Option,
+  Predicate,
   Record,
+  Redacted,
   Result,
   Schedule,
   Schema,
 } from "effect";
-import { HttpClient, HttpClientError, HttpClientResponse } from "effect/http";
-import * as HttpClientRequest from "effect/http/HttpClientRequest";
+import { FetchHttpClient } from "effect/http";
 import { RateLimiter } from "effect/persistence";
 
-const SYSTEM_ONE = "https://api.typesafe.ai/v1/systemone";
-
-const NoulAnswers = Schema.Struct({
-  answers: Schema.Record(Schema.String, Schema.Struct({ noul: Schema.Finite })),
+/** A provider's answers to `noul`s, which the eval's harness reads too, to `choice`s, and to a locate request's both. */
+export const NoulAnswers = Schema.Struct({
+  ...Answered.fields,
+  answers: Schema.Record(Schema.String, Schema.Finite),
 });
 const ChoiceAnswers = Schema.Struct({
-  answers: Schema.Record(Schema.String, Schema.Struct({ choice: Schema.String })),
-});
-/** A locate request's answers: a choice per rule, and a noul per sufficiency question. */
-const LocateAnswers = Schema.Struct({
-  answers: Schema.Record(
-    Schema.String,
-    Schema.Struct({
-      choice: Schema.optionalKey(Schema.String),
-      noul: Schema.optionalKey(Schema.Finite),
-    }),
-  ),
+  ...Answered.fields,
+  answers: Schema.Record(Schema.String, Schema.String),
 });
+const LocateAnswers = Answered;
 
 const refused = (message: string) => JevUnavailable.make({ message });
 
+const messageOf = (cause: unknown) => (cause instanceof Error ? cause.message : String(cause));
+
 /** Bun's API, for a rule's hook. Read off `globalThis`, as presets.ts reads it: Vitest runs this on Node. */
 const bun = (globalThis as { readonly Bun?: unknown }).Bun as Parameters<AppendState>[2];
 
 /**
- * A body as it goes out: when its rule's `reads` names the workspace's
- * packages, with them in its state, and when it has `appendState`, with what
- * the hook returns spread over that. A hook that fails refuses the run.
+ * A prepared body as it goes out: `appendState` sees the shared enriched
+ * state, with what it returns spread over this group's state alone.
+ * A hook that fails refuses the run.
  */
 const hooked = (
-  sent: Body & { readonly state: JevState },
+  body: Body<SystemOneQuestion> & { readonly state: JevState },
   group: Rules,
   file: JudgedFile,
-  workspacePackages: Readonly<Record<string, string>>,
-): Effect.Effect<Body, JevUnavailable> => {
-  const body = Object.values(group).some(
-    (rule) => rule.reads?.includes("workspacePackages") === true,
-  )
-    ? { ...sent, state: { ...sent.state, workspacePackages } }
-    : sent;
+): Effect.Effect<Body<SystemOneQuestion>, JevUnavailable> => {
   const [hook] = Object.entries(group).flatMap(([id, rule]) =>
     rule.appendState === undefined ? [] : [[id, rule.appendState] as const],
   );
@@ -91,71 +89,80 @@ const hooked = (
   return Effect.tryPromise({
     try: async () => ({
       ...body,
-      state: { ...body.state, ...(await appendState(body.state, file, bun)) },
+      state: describeState({ ...body.state, ...(await appendState(body.state, file, bun)) }),
     }),
-    catch: (cause) =>
-      refused(
-        `the appendState of ${id} failed: ${cause instanceof Error ? cause.message : String(cause)}`,
-      ),
+    catch: (cause) => refused(`the appendState of ${id} failed: ${messageOf(cause)}`),
   });
 };
 
-/** A failed attempt, for the log: the status and Ray ID when Jev's side answered, the cause otherwise. */
-const attemptFailure = (
-  problem: HttpClientError.HttpClientError | RateLimiter.RateLimiterError | Cause.TimeoutError,
-): string =>
-  HttpClientError.isHttpClientError(problem) && problem.reason._tag === "StatusCodeError"
-    ? `HTTP ${problem.reason.response.status}, Ray ID ${problem.reason.response.headers["cf-ray"] ?? "none"}`
-    : problem.message;
-
-/** Jev's error body, on one line and cut short, or nothing when it sent none. */
-const detailOf = (body: string): string => {
-  const text = body.trim().replace(/\s+/g, " ");
-  return text.length === 0 ? "" : `: ${text.slice(0, 300)}`;
+/**
+ * A provider's rejection, as the run takes it: a `ContextOverflow` skips the
+ * file, a `RequestBlocked` reports it blocked, and anything else refuses the
+ * run with what the provider said. Read by tag, not class, so a provider
+ * built against another copy of adhere is read the same.
+ */
+const rejected = (cause: unknown): JevOverflow | JevBlocked | JevUnavailable => {
+  if (Predicate.isTagged(cause, "ContextOverflow")) return JevOverflow.make();
+  if (Predicate.isTagged(cause, "RequestBlocked") && Predicate.hasProperty(cause, "reference")) {
+    return JevBlocked.make({ ray: String(cause.reference) });
+  }
+  return refused(messageOf(cause));
 };
 
-/** What Jev answers, with a 400, to a request over its context. */
-const Overflowed = Schema.fromJsonString(
-  Schema.Struct({ detail: Schema.Struct({ error_type: Schema.Literal("max_tokens_exceeded") }) }),
-);
-const isOverflow = (body: string): boolean =>
-  Option.isSome(Schema.decodeUnknownOption(Overflowed)(body));
-
-/** A firewall's block page: HTML where Jev answers JSON, even for its own errors. */
-const isBlockPage = (response: HttpClientResponse.HttpClientResponse, body: string): boolean =>
-  (response.headers["content-type"] ?? "").includes("text/html") ||
-  body.trimStart().startsWith("<");
+/** Whether a failed attempt is worth another: a `RequestFailed`, read by tag, with no answer, a 408, a 429, or a 5xx. */
+const isTransient = (cause: unknown): boolean => {
+  if (!Predicate.isTagged(cause, "RequestFailed") || !Predicate.hasProperty(cause, "status")) {
+    return false;
+  }
+  const { status } = cause;
+  return (
+    status === 0 ||
+    status === 408 ||
+    status === 429 ||
+    (Predicate.isNumber(status) && status >= 500)
+  );
+};
 
 /**
- * A request that failed. A 400 saying the request is over Jev's context is a
- * `JevOverflow`, which skips the file. Any other answer outside 2xx refuses
- * the run with its status and what Jev said, since the status alone does not
- * say why, and a request that got no answer refuses with the cause.
+ * One request to `provider`, as adhere asks every provider: each attempt
+ * after `pace`, given 30 seconds before its signal aborts it, and logged when
+ * it fails. One that failed transiently is tried again up to three times,
+ * after half a second, one, and two. What it answers decodes as `Answers`.
+ * The eval's harness asks its providers this way too.
  */
-const failed = (
-  problem: HttpClientError.HttpClientError | RateLimiter.RateLimiterError | Cause.TimeoutError,
-) => {
-  if (!HttpClientError.isHttpClientError(problem) || problem.reason._tag !== "StatusCodeError") {
-    return Effect.fail(refused(`System One request failed: ${problem.message}`));
-  }
-  const { response } = problem.reason;
-  return Effect.flatMap(
-    Effect.orElseSucceed(response.text, () => "").pipe(
-      Effect.tap((body) =>
-        Effect.logDebug(
-          `Jev answered HTTP ${response.status}, ${response.headers["content-type"] ?? "no content type"}, ${body.length} characters, Ray ID ${response.headers["cf-ray"] ?? "none"}`,
+export const askProvider = <A>(
+  provider: Provider,
+  state: State,
+  questions: ReadonlyArray<Question>,
+  Answers: Schema.Codec<A, unknown, never, never>,
+  apiKey?: string,
+  pace: Effect.Effect<unknown, unknown> = Effect.void,
+): Effect.Effect<A, JevOverflow | JevBlocked | JevUnavailable> =>
+  pace.pipe(
+    Effect.andThen(
+      Effect.tryPromise({
+        try: (signal) =>
+          provider.ask(state, questions, apiKey === undefined ? { signal } : { apiKey, signal }),
+        catch: (cause) => cause,
+      }).pipe(
+        Effect.timeoutOrElse({
+          duration: "30 seconds",
+          orElse: () =>
+            Effect.fail(new RequestFailed(`${provider.model} did not answer in 30 seconds`, 0)),
+        }),
+      ),
+    ),
+    Effect.tapError((cause) => Effect.logDebug(`attempt failed: ${messageOf(cause)}`)),
+    Effect.retry({ while: isTransient, times: 3, schedule: Schedule.exponential("500 millis") }),
+    Effect.mapError(rejected),
+    Effect.flatMap((answered) =>
+      Schema.decodeUnknownEffect(Answers)(answered).pipe(
+        Effect.mapError((problem) =>
+          refused(`${provider.model}'s answers did not decode: ${problem.message}`),
         ),
       ),
-      Effect.tap((body) => Effect.logTrace(`Jev's answer: ${body.slice(0, 4000)}`)),
     ),
-    (body): Effect.Effect<never, JevOverflow | JevBlocked | JevUnavailable> =>
-      response.status === 400 && isOverflow(body)
-        ? Effect.fail(JevOverflow.make())
-        : response.status === 403 && isBlockPage(response, body)
-          ? Effect.fail(JevBlocked.make({ ray: response.headers["cf-ray"] ?? "none given" }))
-          : Effect.fail(refused(`Jev answered HTTP ${response.status}${detailOf(body)}`)),
   );
-};
 
 /**
  * Comparing rules has no file to skip, so a request over Jev's context, or one
@@ -181,70 +188,75 @@ export const JevLive = Layer.effect(Jev)(
   Effect.gen(function* () {
     const config = yield* AdhereConfig;
     const credentials = yield* Credentials;
-    const http = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk);
-    // `--rpm`: one request per 60/rpm seconds, evenly spaced rather than in a
-    // burst each minute. Before the retries, so a retry waits its turn too.
-    type Failure = HttpClientError.HttpClientError | RateLimiter.RateLimiterError;
-    const paced: HttpClient.HttpClient.With<Failure> =
-      config.rpm === undefined
-        ? // The same client, typed to fail the ways the throttled one can.
-          HttpClient.transformResponse(
-            http,
-            (response): Effect.Effect<HttpClientResponse.HttpClientResponse, Failure> => response,
-          )
-        : http.pipe(
-            HttpClient.withRateLimiter({
-              limiter: yield* RateLimiter.RateLimiter,
-              key: "jev",
-              algorithm: "token-bucket",
-              limit: 1,
-              window: Duration.millis(60_000 / config.rpm),
-            }),
+    const fetch = yield* FetchHttpClient.Fetch;
+    const limiter = yield* RateLimiter.RateLimiter;
+
+    const provider: Provider = config.provider ?? jev({ model: config.model, fetch });
+    const previewed = new Set<string>();
+    // The key it asks with, when it names one `adhere login` saves. Read when
+    // a request needs it, not in the layer: a run with nothing pending needs none.
+    const apiKey: Effect.Effect<string | undefined, JevUnavailable> =
+      provider.savedKey === undefined
+        ? Effect.succeed(undefined)
+        : yield* Effect.cached(
+            credentials.key(provider.savedKey).pipe(
+              Effect.map(Redacted.value),
+              Effect.mapError((problem) => refused(problem.message)),
+            ),
           );
-    const client = paced.pipe(
-      HttpClient.transformResponse(Effect.timeout("30 seconds")),
-      // Every failed attempt, including those retried, so a log shows what retrying hid.
-      HttpClient.transformResponse(
-        Effect.tapError((problem) => Effect.logDebug(`attempt failed: ${attemptFailure(problem)}`)),
-      ),
-      HttpClient.retryTransient({
-        schedule: Schedule.exponential("500 millis"),
-        times: 3,
-      }),
-    );
 
-    /** One request. `kind` names the question for the log: judge, locate, blocks, and so on. */
-    const ask = <A>(kind: string, body: Body, Answers: Schema.Codec<A, unknown, never, never>) =>
+    // `--rpm`: one attempt per 60/rpm seconds, evenly spaced rather than in a
+    // burst each minute, so a retry waits its turn too.
+    const pace =
+      config.rpm === undefined
+        ? Effect.void
+        : RateLimiter.sleep(limiter, {
+            key: "provider",
+            algorithm: "token-bucket",
+            limit: 1,
+            window: Duration.millis(60_000 / config.rpm),
+          });
+
+    /** One request. `kind` names the question for the log: judge, locate, conflicts, and so on. */
+    const ask = <A>(
+      kind: string,
+      body: Body<SystemOneQuestion>,
+      Answers: Schema.Codec<A, unknown, never, never>,
+    ) =>
       Effect.gen(function* () {
-        // Read here, not in the layer: a run with nothing pending needs no key.
-        const apiKey = yield* credentials.apiKey.pipe(
-          Effect.mapError((problem) => refused(problem.message)),
-        );
-        const request = yield* HttpClientRequest.bodyJson(
-          HttpClientRequest.post(SYSTEM_ONE),
-          body,
-        ).pipe(Effect.orDie);
+        const key = yield* apiKey;
         const questions = Object.keys(body.questions).length;
         yield* Effect.logDebug(
-          `${kind}: sending ${questions} ${questions === 1 ? "question" : "questions"}, about ${tokensOf(body)} tokens`,
+          `${kind}: asking ${provider.model} ${questions} ${questions === 1 ? "question" : "questions"}, about ${tokensOf(body)} tokens`,
         );
-        const [elapsed, response] = yield* Effect.timed(
-          client.execute(HttpClientRequest.bearerToken(request, apiKey)).pipe(Effect.catch(failed)),
+        if ((kind === "judge" || kind === "locate") && !previewed.has(kind)) {
+          previewed.add(kind);
+          yield* logRequestPayload(
+            `${kind}: first ${kind === "locate" ? "match-to-lines " : ""}request payload`,
+            () => body,
+            "Debug",
+          );
+        }
+        const [elapsed, answered] = yield* Effect.timed(
+          askProvider(provider, body.state, questionsOf(body.questions), Answers, key, pace).pipe(
+            Effect.tapErrorTag("JevOverflow", () =>
+              logRequestPayload(
+                `${kind}: too long request payload (provider overflow)`,
+                () => body,
+              ),
+            ),
+          ),
         );
         yield* Effect.logDebug(
-          `${kind}: HTTP ${response.status} in ${Math.round(Duration.toMillis(elapsed))} ms, Ray ID ${response.headers["cf-ray"] ?? "none"}`,
-        );
-        return yield* HttpClientResponse.schemaBodyJson(Answers)(response).pipe(
-          Effect.mapError((problem) =>
-            refused(`System One response did not decode: ${problem.message}`),
-          ),
+          `${kind}: ${provider.model} answered in ${Math.round(Duration.toMillis(elapsed))} ms`,
         );
+        return answered;
       });
 
     /** Every answer to a body, over as many requests as Jev's context needs. */
     const answersTo = <A>(
       kind: string,
-      body: Body,
+      body: Body<SystemOneQuestion>,
       Answers: Schema.Codec<
         { readonly answers: Readonly<Record<string, A>> },
         unknown,
@@ -269,7 +281,7 @@ export const JevLive = Layer.effect(Jev)(
     const answersAbout = <A>(
       kind: string,
       rules: Rules,
-      bodyOf: (group: Rules) => Body & { readonly state: JevState },
+      bodyOf: (group: Rules) => Body<SystemOneQuestion> & { readonly state: JevState },
       file: JudgedFile,
       Answers: Schema.Codec<
         { readonly answers: Readonly<Record<string, A>> },
@@ -280,9 +292,8 @@ export const JevLive = Layer.effect(Jev)(
     ) =>
       Effect.map(
         Effect.forEach(requestGroups(rules), (group) =>
-          Effect.flatMap(
-            hooked(bodyOf(group), group, file, config.workspacePackages ?? {}),
-            (body) => answersTo(kind, body, Answers),
+          Effect.flatMap(hooked(bodyOf(group), group, file), (body) =>
+            answersTo(kind, body, Answers),
           ),
         ),
         (groups) =>
@@ -294,46 +305,47 @@ export const JevLive = Layer.effect(Jev)(
       rules: Rules,
       sampled: ReadonlyArray<RuleId>,
       file: JudgedFile,
+      readState: ReadState = {},
     ) {
       const answers = yield* answersAbout(
         "judge",
         rules,
-        (group) => judgeBody(config.model, lines, group, sampled),
+        (group) => judgeBody(config.model, lines, group, sampled, readState),
         file,
         NoulAnswers,
       );
-      /** Each rule's answer, read from the question it rode under. */
-      const answered = (ids: ReadonlyArray<RuleId>, keyOf: (id: RuleId) => string) =>
-        Record.fromEntries(
-          ids.flatMap((id) => {
-            const answer = answers[keyOf(id)];
-            return answer === undefined ? [] : [[id, answer.noul] as const];
-          }),
-        );
-      // A matcher Jev left unanswered counts as a no.
-      const scored = (
-        kind: keyof MatcherScores,
-        id: RuleId,
-        matchers: ReadonlyArray<string> = [],
-      ) => matchers.map((_, index) => answers[matcherKey(kind, id, index)]?.noul ?? 0);
+      const all = (scores: ReadonlyArray<number | undefined>): scores is ReadonlyArray<number> =>
+        scores.every((score) => score !== undefined);
+      // A rule is answered when its question and each of its matchers' are: a
+      // matcher left unanswered, as a refusal leaves one, leaves its rule
+      // unanswered too, to ask again on the next run rather than cache as a no.
+      const judged = Object.entries(rules).flatMap(([id, rule]) => {
+        const scored = (kind: keyof MatcherScores) =>
+          (rule[kind] ?? []).map((_, index) => answers[matcherKey(kind, id, index)]);
+        const probability = answers[id];
+        const appliesTo = scored("appliesTo");
+        const excludeIf = scored("excludeIf");
+        return probability === undefined || !all(appliesTo) || !all(excludeIf)
+          ? []
+          : [{ id, rule, probability, appliesTo, excludeIf }];
+      });
       const matchers: Record<RuleId, MatcherScores> = Record.fromEntries(
-        Object.entries(rules).flatMap(([id, rule]) =>
+        judged.flatMap(({ id, rule, appliesTo, excludeIf }) =>
           rule.appliesTo === undefined && rule.excludeIf === undefined
             ? []
-            : [
-                [
-                  id,
-                  {
-                    appliesTo: scored("appliesTo", id, rule.appliesTo),
-                    excludeIf: scored("excludeIf", id, rule.excludeIf),
-                  },
-                ] as const,
-              ],
+            : [[id, { appliesTo, excludeIf }] as const],
         ),
       );
       return {
-        probabilities: answered(Object.keys(rules), (id) => id),
-        linter: answered(sampled, linterKey),
+        probabilities: Record.fromEntries(
+          judged.map(({ id, probability }) => [id, probability] as const),
+        ),
+        linter: Record.fromEntries(
+          sampled.flatMap((id) => {
+            const answer = answers[linterKey(id)];
+            return answer === undefined ? [] : [[id, answer] as const];
+          }),
+        ),
         ...(Record.isEmptyRecord(matchers) ? {} : { matchers }),
       } satisfies Judged;
     });
@@ -342,25 +354,26 @@ export const JevLive = Layer.effect(Jev)(
       lines: Lines,
       rules: Rules,
       file: JudgedFile,
+      readState: ReadState = {},
     ) {
       const sections = sectionsOf(lines);
       const answers = yield* answersAbout(
         "locate",
         rules,
-        (group) => locateBody(config.model, lines, group),
+        (group) => locateBody(config.model, lines, group, readState),
         file,
         LocateAnswers,
       );
       // A file of one section has every finding in it. An answer that names no section,
-      // or none at all, which Jev should not give, leaves the rule unlocated; lines outside
-      // the section leave it without lines.
+      // or none at all, which a model should not give, leaves the rule unlocated; lines
+      // outside the section leave it without lines.
       return Record.filterMap(rules, (_, id) => {
-        const chosen = sections.length === 1 ? 1 : Number(answers[id]?.choice);
+        const chosen = sections.length === 1 ? 1 : Number(answers[id]);
         const section = sections[chosen - 1];
-        const sufficiency = answers[sufficiencyKey(id)]?.noul;
-        if (section === undefined || sufficiency === undefined) return Result.failVoid;
+        const sufficiency = answers[sufficiencyKey(id)];
+        if (section === undefined || typeof sufficiency !== "number") return Result.failVoid;
         const [start, end] = (["start", "end"] as const).map((edge) =>
-          Number(answers[edgeKey(edge, id, chosen)]?.choice),
+          Number(answers[edgeKey(edge, id, chosen)]),
         );
         const within = (line: number | undefined): line is number =>
           line !== undefined && section.first <= line && line <= section.last;