@drkmttr/adhere 0.15.0 → 0.17.5

package/src/providers/post.ts

@@ -0,0 +1,53 @@
+/** How each built-in provider sends a request and reads what comes back. */
+import { RequestFailed } from "#providers/provider.ts";
+import { Schema } from "effect";
+
+const messageOf = (cause: unknown) => (cause instanceof Error ? cause.message : String(cause));
+
+/**
+ * POSTs `body` as JSON, once: adhere tries again. A request that gets no
+ * answer, or whose answer breaks off, throws a `RequestFailed` with status 0.
+ */
+export const post = async (
+  name: string,
+  url: string,
+  headers: Readonly<Record<string, string>>,
+  body: unknown,
+  signal: AbortSignal,
+  fetch: typeof globalThis.fetch = globalThis.fetch,
+): Promise<{ readonly response: Response; readonly text: string }> => {
+  try {
+    const response = await fetch(url, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", ...headers },
+      body: JSON.stringify(body),
+      signal,
+    });
+    return { response, text: await response.text() };
+  } catch (cause) {
+    throw new RequestFailed(`${name} did not answer: ${messageOf(cause)}`, 0);
+  }
+};
+
+/**
+ * A response that is not ok, as the `RequestFailed` to throw: its status, the
+ * Ray ID of the Cloudflare edge in front of the API when there is one, and
+ * its body on one line and cut short.
+ */
+export const failed = (name: string, response: Response, text: string): RequestFailed => {
+  const ray = response.headers.get("cf-ray");
+  const detail = text.trim().replace(/\s+/g, " ").slice(0, 300);
+  return new RequestFailed(
+    `${name} answered HTTP ${response.status}${ray === null ? "" : `, Ray ID ${ray}`}${detail === "" ? "" : `: ${detail}`}`,
+    response.status,
+  );
+};
+
+/** An ok response's body, decoded by `schema`. One that does not decode throws, naming who sent it. */
+export const decoded = <A>(name: string, schema: Schema.Codec<A, string>, text: string): A => {
+  try {
+    return Schema.decodeUnknownSync(schema)(text);
+  } catch (cause) {
+    throw new Error(`${name}'s answer did not decode: ${messageOf(cause)}`);
+  }
+};

package/src/providers/provider.ts

@@ -0,0 +1,131 @@
+/**
+ * What adhere asks a decision model, and what it takes back. A provider puts
+ * adhere's questions to its model in whatever form the model takes; adhere
+ * builds the questions, splits them into requests, paces and retries them,
+ * and reads the answers. Jev's provider is the default.
+ */
+import { Schema } from "effect";
+
+/**
+ * What the questions ask about: the file's code by section under `code`,
+ * beside what adhere read for it; comparing rules, the rules under `rules`.
+ */
+export type State = Readonly<Record<string, unknown>>;
+
+/**
+ * One of adhere's questions, as adhere phrases it. `question` names what it
+ * asks about in backticks: keys of the state, such as `code`, and of
+ * `fields`, such as `rule`, `must`, `never`, or `matcher`, which hold the
+ * rule it asks about and its examples. A `noul` asks whether something holds,
+ * `yes` and `no` saying what each answer means when adhere says. A `choice`
+ * asks for the key of one of its `options`, each beside what it means when
+ * its key does not say; it can have a single option.
+ */
+export type Question =
+  | {
+      readonly id: string;
+      readonly type: "noul";
+      readonly question: string;
+      readonly fields: Readonly<Record<string, unknown>>;
+      readonly yes?: string;
+      readonly no?: string;
+    }
+  | {
+      readonly id: string;
+      readonly type: "choice";
+      readonly question: string;
+      readonly fields: Readonly<Record<string, unknown>>;
+      readonly options: Readonly<Record<string, string | null>>;
+    };
+
+/**
+ * A provider's answers by question id: a `noul`'s probability that it holds,
+ * from 0 to 1, or the key of a `choice`'s option. An id left out goes
+ * unanswered: its rule is asked again on the next run, or its finding not
+ * located. `inputTokens` and `model`, what the model read and which version
+ * answered, are for the record, such as the eval's costs.
+ */
+export const Answered = Schema.Struct({
+  answers: Schema.Record(Schema.String, Schema.Union([Schema.Finite, Schema.String])),
+  inputTokens: Schema.optional(Schema.Finite),
+  model: Schema.optional(Schema.String),
+});
+export interface Answered extends Schema.Schema.Type<typeof Answered> {}
+
+/** The keys `adhere login` saves: TypeSafe AI's, for Jev, and with `--openai`, OpenAI's. */
+export const SAVED_KEYS = ["typesafe", "openai"] as const;
+export type SavedKey = (typeof SAVED_KEYS)[number];
+
+/**
+ * What adhere passes each call of `ask`: the key, when the provider names a
+ * `savedKey`, and the signal that aborts the call when adhere gives up on it.
+ */
+export interface AskOptions {
+  readonly apiKey?: string;
+  readonly signal: AbortSignal;
+}
+
+/**
+ * Where adhere's questions go. `ask` answers one request's questions. Its
+ * rejection stops the run, as Jev being down does, but for three:
+ * `ContextOverflow` skips the file, `RequestBlocked` reports it blocked, and
+ * `RequestFailed` is tried again when its status says it may pass. adhere
+ * paces each call with `--rpm`, and gives up on one after 30 seconds. `model`
+ * names what it asks: adhere's cache keeps answers by it, so a new model is
+ * asked again. A provider that names a `savedKey` gets that key as `apiKey`:
+ * its environment variable when set, otherwise the one `adhere login` saved.
+ * adhere calls `ask` on the provider, so a class can implement it.
+ */
+export interface Provider {
+  readonly model: string;
+  readonly savedKey?: SavedKey;
+  ask(state: State, questions: ReadonlyArray<Question>, options: AskOptions): Promise<Answered>;
+}
+
+/**
+ * Splits off the choices of a single option, which some APIs refuse to be
+ * asked: each is answered by its option, and the rest are left to ask.
+ */
+export const withoutSingleChoices = (questions: ReadonlyArray<Question>) => {
+  const answers: Record<string, number | string> = {};
+  const asked = questions.filter((question) => {
+    const [only, ...others] = question.type === "choice" ? Object.keys(question.options) : [];
+    if (only === undefined || others.length > 0) return true;
+    answers[question.id] = only;
+    return false;
+  });
+  return { answers, asked };
+};
+
+/** Thrown from `ask` when the request is longer than the model takes: adhere skips the file. */
+export class ContextOverflow extends Error {
+  readonly _tag = "ContextOverflow";
+}
+
+/**
+ * Thrown from `ask` when something in front of the model refused the request
+ * for what it holds: adhere reports the file blocked, with `reference` to look
+ * the block up by, and goes on.
+ */
+export class RequestBlocked extends Error {
+  readonly _tag = "RequestBlocked";
+  constructor(readonly reference: string) {
+    super(`blocked, reference ${reference}`);
+  }
+}
+
+/**
+ * Thrown from `ask` when the request failed: `status` is the HTTP status it
+ * got, or 0 when it got no answer. adhere tries one with no answer, a 408, a
+ * 429, or a 5xx again, up to three times, after half a second, one, and two;
+ * any other stops the run with its message.
+ */
+export class RequestFailed extends Error {
+  readonly _tag = "RequestFailed";
+  constructor(
+    message: string,
+    readonly status: number,
+  ) {
+    super(message);
+  }
+}

package/src/providers/providers.test.ts

@@ -0,0 +1,276 @@
+import type { Rule } from "#config.ts";
+import { cloudflare } from "#providers/cloudflare.ts";
+import { jev } from "#providers/jev.ts";
+import { openai } from "#providers/openai.ts";
+import { ContextOverflow, type Question, RequestFailed } from "#providers/provider.ts";
+import { questionsOf, requestOf } from "#providers/systemOne.ts";
+import { conflictBody, contradictBody, judgeBody, locateBody } from "#services/Jev.ts";
+import { describe, expect, it } from "vite-plus/test";
+
+const plain: Rule = {
+  description: "Ports must be named.",
+  must: "const port = PORT;",
+  never: "listen(3000)",
+  appliesTo: ["a server's entry point"],
+};
+const lines = ["const port = 3000;", "", "listen(port);"];
+const compared = [
+  { id: "a", rule: plain },
+  { id: "b", rule: { description: "Ports must be literals.", must: "listen(3000)" } },
+];
+
+/** A fetch that records each request's URL, headers, and body, and answers with `respond`. */
+const recording = (respond: (body: Record<string, unknown>, sent: number) => Response) => {
+  const sent: Array<{ url: string; authorization: string | null; body: Record<string, unknown> }> =
+    [];
+  const fetch = (async (url: unknown, init?: RequestInit) => {
+    const body = JSON.parse(String(init?.body)) as Record<string, unknown>;
+    sent.push({
+      url: String(url),
+      authorization: new Headers(init?.headers).get("authorization"),
+      body,
+    });
+    return respond(body, sent.length);
+  }) as typeof globalThis.fetch;
+  return { sent, fetch };
+};
+
+const noul = (id: string): Question => ({ id, type: "noul", question: `Is ${id} so?`, fields: {} });
+
+/** What adhere passes `ask` beside the key: a signal no test aborts. */
+const signal = new AbortController().signal;
+
+describe("System One", () => {
+  it("puts adhere's questions back as adhere built them, for Jev and Clef", () => {
+    const bodies = [
+      judgeBody("jev-latest", lines, { plain }, ["plain"]),
+      locateBody("jev-latest", lines, { plain }),
+      conflictBody("jev-latest", compared, [[1], [0]]),
+      contradictBody("jev-latest", compared, [0, 1]),
+    ];
+    for (const body of bodies) {
+      const rebuilt = requestOf(body.model, body.state, questionsOf(body.questions));
+      expect(JSON.stringify(rebuilt)).toBe(JSON.stringify(body));
+    }
+  });
+});
+
+describe("jev", () => {
+  it("sends adhere's request to TypeSafe with the key, and reads each answer as a value", async () => {
+    const body = judgeBody("jev-latest", lines, { plain });
+    const { sent, fetch } = recording(() =>
+      Response.json({
+        model: "jev-1.13.0",
+        answers: { plain: { noul: 0.91 }, "appliesTo:plain:0": { noul: 0.2 } },
+        usage: { input_tokens: 420 },
+      }),
+    );
+    const answered = await jev({ apiKey: "tsk_test", fetch }).ask(
+      body.state,
+      questionsOf(body.questions),
+      { signal },
+    );
+    expect(sent.map(({ url, authorization }) => [url, authorization])).toEqual([
+      ["https://api.typesafe.ai/v1/systemone", "Bearer tsk_test"],
+    ]);
+    expect(JSON.stringify(sent[0]?.body)).toBe(JSON.stringify(body));
+    expect(answered).toEqual({
+      answers: { plain: 0.91, "appliesTo:plain:0": 0.2 },
+      inputTokens: 420,
+      model: "jev-1.13.0",
+    });
+  });
+});
+
+describe("saved keys", () => {
+  it("are named by jev and openai built without a key, and asked with the key adhere passes", async () => {
+    const toJev = recording(() => Response.json({ answers: {} }));
+    const toOpenAI = recording(() => Response.json({ answers: [] }));
+    const jevProvider = jev({ fetch: toJev.fetch });
+    const openaiProvider = openai({ fetch: toOpenAI.fetch });
+    expect([jevProvider.savedKey, openaiProvider.savedKey]).toEqual(["typesafe", "openai"]);
+    expect(jev({ apiKey: "tsk_own" }).savedKey).toBeUndefined();
+    await jevProvider.ask({}, [noul("a")], { apiKey: "tsk_saved", signal });
+    await openaiProvider.ask({}, [noul("a")], { apiKey: "sk-saved", signal });
+    expect([...toJev.sent, ...toOpenAI.sent].map(({ authorization }) => authorization)).toEqual([
+      "Bearer tsk_saved",
+      "Bearer sk-saved",
+    ]);
+    await expect(jevProvider.ask({}, [noul("a")], { signal })).rejects.toThrow(
+      "No TypeSafe AI API key",
+    );
+  });
+});
+
+describe("cloudflare", () => {
+  it("asks 64 questions to a request under ids Clef takes, answers a choice of one option unasked, and puts each answer back under its key", async () => {
+    const questions: ReadonlyArray<Question> = [
+      ...Array.from({ length: 70 }, (_, index) => noul(`linter:rule/${index}`)),
+      { id: "start:rule:1", type: "choice", question: "Where?", fields: {}, options: { "3": "x" } },
+    ];
+    const { sent, fetch } = recording((body) =>
+      Response.json({
+        result: {
+          model: "clef",
+          answers: Object.fromEntries(
+            Object.keys(body.questions as object).map((id) => [id, { type: "noul", noul: 0.5 }]),
+          ),
+          usage: { input_tokens: 100 },
+        },
+      }),
+    );
+    const answered = await cloudflare({ accountId: "acct", apiToken: "cft", fetch }).ask(
+      { code: { "1": "x" } },
+      questions,
+      { signal },
+    );
+    expect(sent.map(({ url }) => url)).toEqual([
+      "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef",
+      "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef",
+    ]);
+    expect(sent.map(({ body }) => Object.keys(body.questions as object).length)).toEqual([64, 6]);
+    expect(Object.keys(sent[1]?.body.questions as object)).toEqual([
+      "q0",
+      "q1",
+      "q2",
+      "q3",
+      "q4",
+      "q5",
+    ]);
+    expect(answered.answers["linter:rule/69"]).toBe(0.5);
+    expect(answered.answers["start:rule:1"]).toBe("3");
+    expect(answered.inputTokens).toBe(200);
+  });
+
+  it("skips as over its context a request it read to its limit, or one Workers AI refused as too large", async () => {
+    const read = (tokens: number) =>
+      recording(() =>
+        Response.json({
+          result: { answers: { q0: { noul: 0.5 } }, usage: { input_tokens: tokens } },
+        }),
+      ).fetch;
+    const flash = (fetch: typeof globalThis.fetch) =>
+      cloudflare({ accountId: "acct", apiToken: "cft", model: "clef-flash", fetch }).ask(
+        {},
+        [noul("a")],
+        { signal },
+      );
+    expect((await flash(read(23_999))).answers).toEqual({ a: 0.5 });
+    await expect(flash(read(24_000))).rejects.toBeInstanceOf(ContextOverflow);
+    const tooLarge = recording(() => new Response("{}", { status: 413 })).fetch;
+    await expect(flash(tooLarge)).rejects.toBeInstanceOf(ContextOverflow);
+  });
+});
+
+describe("openai", () => {
+  it("asks each question as a decision question, its parts in JSON, and leaves a refusal unanswered", async () => {
+    const questions: ReadonlyArray<Question> = [
+      {
+        id: "plain",
+        type: "noul",
+        question: "Does `code` diverge from `must`?",
+        fields: { rule: "Ports must be named.", must: "const port = PORT;" },
+        yes: "it diverges",
+        no: "it follows",
+      },
+      {
+        id: "plain:section",
+        type: "choice",
+        question: "Which?",
+        fields: {},
+        options: { "1": null, "2": "the second" },
+      },
+      {
+        id: "start:plain:1",
+        type: "choice",
+        question: "Where?",
+        fields: {},
+        options: { "7": "x" },
+      },
+      noul("refused"),
+    ];
+    const { sent, fetch } = recording(() =>
+      Response.json({
+        model: "gpt-6-luna",
+        answers: [
+          { type: "predicate", name: "plain", probability: 0.9 },
+          {
+            type: "choice",
+            name: "plain:section",
+            choice: "2",
+            probabilities: [],
+            confidence: 0.8,
+          },
+          { type: "refusal", name: "refused" },
+        ],
+        usage: { input_tokens: 300 },
+      }),
+    );
+    const answered = await openai({ apiKey: "sk-test", fetch }).ask(
+      { code: { "1": "x" } },
+      questions,
+      { signal },
+    );
+    expect(sent[0]?.body).toEqual({
+      model: "gpt-6-luna",
+      input: JSON.stringify({ code: { "1": "x" } }),
+      questions: [
+        {
+          type: "predicate",
+          name: "plain",
+          instructions: JSON.stringify({
+            instructions: {
+              question: "Does `code` diverge from `must`?",
+              rule: "Ports must be named.",
+              must: "const port = PORT;",
+            },
+            criteria: { true: "it diverges", false: "it follows" },
+          }),
+        },
+        {
+          type: "choice",
+          name: "plain:section",
+          instructions: "Which?",
+          choices: [{ value: "1" }, { value: "2", description: "the second" }],
+        },
+        { type: "predicate", name: "refused", instructions: "Is refused so?" },
+      ],
+    });
+    expect(answered).toEqual({
+      answers: { "start:plain:1": "7", plain: 0.9, "plain:section": "2" },
+      inputTokens: 300,
+      model: "gpt-6-luna",
+    });
+  });
+});
+
+describe("a failed request", () => {
+  it("is thrown once, as a RequestFailed with its status and Ray ID, or status 0 when nothing answered", async () => {
+    const busy = recording(
+      () => new Response("busy\n", { status: 503, headers: { "cf-ray": "8c1f-LAX" } }),
+    );
+    const failed = await jev({ apiKey: "tsk_test", fetch: busy.fetch })
+      .ask({}, [noul("a")], { signal })
+      .catch((cause: unknown) => cause);
+    expect(failed).toBeInstanceOf(RequestFailed);
+    expect(failed).toMatchObject({
+      status: 503,
+      message: "Jev answered HTTP 503, Ray ID 8c1f-LAX: busy",
+    });
+    expect(busy.sent).toHaveLength(1);
+
+    const down = (async () => {
+      throw new TypeError("fetch failed");
+    }) as unknown as typeof globalThis.fetch;
+    await expect(
+      openai({ apiKey: "sk-test", fetch: down }).ask({}, [noul("a")], { signal }),
+    ).rejects.toMatchObject({ status: 0, message: "OpenAI did not answer: fetch failed" });
+  });
+
+  it("is an answer that does not decode, naming who sent it", async () => {
+    const { fetch } = recording(() => Response.json({ answers: "none" }));
+    await expect(
+      openai({ apiKey: "sk-test", fetch }).ask({}, [noul("a")], { signal }),
+    ).rejects.toThrow("OpenAI's answer did not decode");
+  });
+});

package/src/providers/systemOne.ts

@@ -0,0 +1,95 @@
+/**
+ * System One, the format Jev takes and Clef follows, and adhere's questions
+ * in it. adhere builds its questions this way, as its cache and Jev have
+ * always read them; a provider gets them split into their parts by
+ * `questionsOf`, and Jev's and Clef's put them back with `requestOf`.
+ */
+import type { Answered, Question, State } from "#providers/provider.ts";
+import { Schema } from "effect";
+
+/** A question's instructions: the question alone, or beside the fields it names. */
+type Instructions = string | (Readonly<Record<string, unknown>> & { readonly question: string });
+
+/** A question as System One takes it: a `noul` with what yes and no mean, or a `choice` among its criteria's keys. */
+export type SystemOneQuestion =
+  | {
+      readonly type: "noul";
+      readonly instructions: Instructions;
+      readonly criteria?: { readonly true: string; readonly false: string };
+    }
+  | {
+      readonly type: "choice";
+      readonly instructions: Instructions;
+      readonly criteria: Readonly<Record<string, string | null>>;
+    };
+
+/** A request as System One takes it: questions by id about `state`, its questions typed as `Q`. */
+export interface SystemOneRequest<Q = SystemOneQuestion> {
+  readonly model: string;
+  readonly state: State;
+  readonly questions: Readonly<Record<string, Q>>;
+}
+
+/**
+ * What System One answers: by id, a `noul`'s probability or a `choice`'s
+ * key, with the model that answered and the tokens it read.
+ */
+export const SystemOneAnswered = Schema.Struct({
+  answers: Schema.Record(
+    Schema.String,
+    Schema.Struct({
+      noul: Schema.optionalKey(Schema.Finite),
+      choice: Schema.optionalKey(Schema.String),
+    }),
+  ),
+  model: Schema.optionalKey(Schema.String),
+  usage: Schema.optionalKey(Schema.Struct({ input_tokens: Schema.Finite })),
+});
+
+/** Each question of a request split into what it asks, the fields it names, and what its answers mean. */
+export const questionsOf = (
+  questions: Readonly<Record<string, SystemOneQuestion>>,
+): ReadonlyArray<Question> =>
+  Object.entries(questions).map(([id, asked]) => {
+    const { question, ...fields } =
+      typeof asked.instructions === "string"
+        ? { question: asked.instructions }
+        : asked.instructions;
+    if (asked.type === "choice")
+      return { id, type: "choice", question, fields, options: asked.criteria };
+    return asked.criteria === undefined
+      ? { id, type: "noul", question, fields }
+      : { id, type: "noul", question, fields, yes: asked.criteria.true, no: asked.criteria.false };
+  });
+
+/** A question as System One asks it, as adhere built it: a question with no fields is its instructions alone. */
+export const systemOneQuestionOf = (asked: Question): SystemOneQuestion => {
+  const instructions =
+    Object.keys(asked.fields).length === 0
+      ? asked.question
+      : { question: asked.question, ...asked.fields };
+  if (asked.type === "choice") return { type: "choice", instructions, criteria: asked.options };
+  return asked.yes === undefined || asked.no === undefined
+    ? { type: "noul", instructions }
+    : { type: "noul", instructions, criteria: { true: asked.yes, false: asked.no } };
+};
+
+/** A request as System One takes it, each question by its id. */
+export const requestOf = (
+  model: string,
+  state: State,
+  questions: ReadonlyArray<Question>,
+): SystemOneRequest => ({
+  model,
+  state,
+  questions: Object.fromEntries(questions.map((asked) => [asked.id, systemOneQuestionOf(asked)])),
+});
+
+/** System One's answers as a provider gives them: each a probability or a key. One with neither is left out. */
+export const answersOf = (answers: typeof SystemOneAnswered.Type.answers): Answered["answers"] =>
+  Object.fromEntries(
+    Object.entries(answers).flatMap(
+      ([id, { noul, choice }]): ReadonlyArray<readonly [string, number | string]> =>
+        noul !== undefined ? [[id, noul]] : choice !== undefined ? [[id, choice]] : [],
+    ),
+  );

package/src/services/AdhereConfig.ts

@@ -131,10 +131,8 @@ export const AdhereConfigLive = (flags: Flags) =>
         }
       }
       const scopedRules = withOverrides(entries, overrides);
-      // The manifests are read on every run, so a changed workspace is seen, and only when a rule reads them.
-      const workspacePackages = scopedRules.some(
-        (entry) => entry.rule.reads?.includes("workspacePackages") === true,
-      )
+      // Read on every run so workspace edits are seen, but only when the config enables this context.
+      const workspacePackages = resolved.reads.includes("workspacePackages")
         ? yield* workspacePackagesOf(cwd)
         : undefined;
       return AdhereConfig.of({