@cursor/bdk 0.2.1

package/src/extensions/jev/lib/evaluate.ts

@@ -0,0 +1,266 @@
+/**
+ * System One decisions for Agent SDK host code.
+ *
+ * Chat models write. {@link evaluate} asks Jev for typed choice / score /
+ * boolean answers over shared state. Call it from a project tool, a
+ * router, or other host code after mounting `@cursor/bdk/extensions/jev`.
+ * Do not set Jev as the `defineAgent` turn model.
+ *
+ * ```ts
+ * import { evaluate, decide, above, needsHuman } from "@cursor/bdk/extensions/jev";
+ *
+ * const { answers } = await evaluate({
+ *   state: draft,
+ *   questions: {
+ *     route: {
+ *       type: "choice",
+ *       instructions: "Which queue should handle this?",
+ *       criteria: {
+ *         billing: "charges, invoices, refunds",
+ *         shipping: "delivery",
+ *         technical: "bugs and outages",
+ *       },
+ *     },
+ *     post: {
+ *       type: "boolean",
+ *       instructions: "Is this safe to post publicly?",
+ *     },
+ *   },
+ * });
+ *
+ * if (needsHuman(answers.route) || !above(answers.post, 0.85)) {
+ *   return { action: "review" };
+ * }
+ * ```
+ *
+ * Auth is `TYPESAFE_API_KEY`. `model` is a TypeSafe id (`jev-latest` by
+ * default).
+ */
+
+/** TypeSafe model used when {@link EvaluateArgs.model} is omitted. */
+export const DEFAULT_EVALUATION_MODEL = "jev-latest";
+
+/** Default bar for {@link needsHuman} when the caller does not pass one. */
+export const DEFAULT_HUMAN_THRESHOLD = 0.7;
+
+const DEFAULT_BASE_URL = "https://api.typesafe.ai/v1";
+
+export type EvaluationState = string | Record<string, unknown> | readonly unknown[];
+
+export type BooleanQuestion = {
+  type: "boolean";
+  instructions: string;
+};
+
+export type ChoiceCriterion = string | readonly string[] | { includes: readonly string[] } | null;
+
+export type ChoiceQuestion<
+  C extends Record<string, ChoiceCriterion> = Record<string, ChoiceCriterion>,
+> = {
+  type: "choice";
+  instructions: string;
+  criteria: C;
+};
+
+export type ScoreQuestion = {
+  type: "score";
+  instructions: string;
+  criteria: readonly string[];
+};
+
+export type EvaluationQuestion = BooleanQuestion | ChoiceQuestion | ScoreQuestion;
+
+export type QuestionMap = Record<string, EvaluationQuestion>;
+
+export type BooleanAnswer = {
+  type: "boolean";
+  probability: number;
+};
+
+export type ChoiceAnswer<C extends Record<string, unknown> = Record<string, unknown>> = {
+  type: "choice";
+  choice: keyof C & string;
+  probabilities: { [K in keyof C]: number };
+};
+
+export type ScoreAnswer = {
+  type: "score";
+  score: number;
+  probabilities: Record<string, number>;
+};
+
+export type EvaluationAnswer = BooleanAnswer | ChoiceAnswer | ScoreAnswer;
+
+export type AnswerFor<Q> = Q extends BooleanQuestion
+  ? BooleanAnswer
+  : Q extends ChoiceQuestion<infer C>
+    ? ChoiceAnswer<C>
+    : Q extends ScoreQuestion
+      ? ScoreAnswer
+      : never;
+
+export type AnswersFor<Q extends QuestionMap> = {
+  [K in keyof Q]: AnswerFor<Q[K]>;
+};
+
+export type EvaluateResult<Q extends QuestionMap = QuestionMap> = {
+  answers: AnswersFor<Q>;
+};
+
+export type EvaluateArgs<Q extends QuestionMap = QuestionMap> = {
+  model?: string;
+  state: EvaluationState;
+  questions: Q;
+  apiKey?: string;
+  baseUrl?: string;
+  fetch?: typeof globalThis.fetch;
+};
+
+type WireQuestion =
+  | { type: "noul"; instructions: string }
+  | {
+      type: "choice";
+      instructions: string;
+      criteria: ChoiceQuestion["criteria"];
+    }
+  | { type: "score"; instructions: string; criteria: readonly string[] };
+
+function resolveApiKey(explicit: string | undefined): string {
+  const key = (explicit ?? process.env.TYPESAFE_API_KEY)?.trim();
+  if (key === undefined || key === "") {
+    throw new Error("evaluate requires TYPESAFE_API_KEY");
+  }
+  return key;
+}
+
+function toWireQuestion(question: EvaluationQuestion): WireQuestion {
+  if (question.type === "boolean") {
+    return { type: "noul", instructions: question.instructions };
+  }
+  return question;
+}
+
+function asRecord(value: unknown, label: string): Record<string, unknown> {
+  if (value === null || typeof value !== "object") {
+    throw new Error(`evaluate ${label}`);
+  }
+  return value as Record<string, unknown>;
+}
+
+function decodeAnswer(question: EvaluationQuestion, raw: unknown): EvaluationAnswer {
+  const value = asRecord(raw, "answer is missing");
+  if (question.type === "boolean") {
+    if (typeof value.noul !== "number") {
+      throw new Error("evaluate noul is not a number");
+    }
+    return { type: "boolean", probability: value.noul };
+  }
+  if (question.type === "choice") {
+    if (typeof value.choice !== "string") {
+      throw new Error("evaluate choice is not a string");
+    }
+    return {
+      type: "choice",
+      choice: value.choice,
+      probabilities: asRecord(
+        value.probabilities,
+        "choice probabilities are missing",
+      ) as ChoiceAnswer["probabilities"],
+    };
+  }
+  if (typeof value.score !== "number") {
+    throw new Error("evaluate score is not a number");
+  }
+  return {
+    type: "score",
+    score: value.score,
+    probabilities: asRecord(
+      value.probabilities,
+      "score probabilities are missing",
+    ) as ScoreAnswer["probabilities"],
+  };
+}
+
+async function postSystemOne(args: {
+  url: string;
+  apiKey: string;
+  body: {
+    model: string;
+    state: EvaluationState;
+    questions: Record<string, WireQuestion>;
+  };
+  fetch: typeof globalThis.fetch;
+}): Promise<Record<string, unknown>> {
+  const response = await args.fetch(args.url, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${args.apiKey}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(args.body),
+  });
+  const text = await response.text();
+  let parsed: unknown;
+  try {
+    parsed = text.length === 0 ? {} : JSON.parse(text);
+  } catch {
+    throw new Error(`evaluate returned non-JSON (${response.status})`);
+  }
+  if (!response.ok) {
+    throw new Error(`evaluate failed (${response.status}): ${text.slice(0, 240)}`);
+  }
+  return asRecord(
+    asRecord(parsed, "response is not an object").answers,
+    "response is missing answers",
+  );
+}
+
+/** Ask Jev for typed answers over one state. */
+export async function evaluate<const Q extends QuestionMap>(
+  args: EvaluateArgs<Q>,
+): Promise<EvaluateResult<Q>> {
+  const questions: Record<string, WireQuestion> = {};
+  for (const [id, question] of Object.entries(args.questions)) {
+    questions[id] = toWireQuestion(question);
+  }
+  const raw = await postSystemOne({
+    url: `${(args.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/u, "")}/systemone`,
+    apiKey: resolveApiKey(args.apiKey),
+    fetch: args.fetch ?? globalThis.fetch,
+    body: {
+      model: args.model ?? DEFAULT_EVALUATION_MODEL,
+      state: args.state,
+      questions,
+    },
+  });
+  const answers = {} as AnswersFor<Q>;
+  for (const id of Object.keys(args.questions) as (keyof Q & string)[]) {
+    answers[id] = decodeAnswer(args.questions[id], raw[id]) as AnswersFor<Q>[typeof id];
+  }
+  return { answers };
+}
+
+/** Same as {@link evaluate}, returning only `answers`. */
+export async function decide<const Q extends QuestionMap>(
+  args: EvaluateArgs<Q>,
+): Promise<AnswersFor<Q>> {
+  const result = await evaluate(args);
+  return result.answers;
+}
+
+export function above(answer: BooleanAnswer | ScoreAnswer, threshold: number): boolean {
+  if (answer.type === "boolean") {
+    return answer.probability >= threshold;
+  }
+  return answer.score >= threshold;
+}
+
+export function needsHuman(
+  answer: BooleanAnswer | ChoiceAnswer,
+  threshold: number = DEFAULT_HUMAN_THRESHOLD,
+): boolean {
+  if (answer.type === "boolean") {
+    return answer.probability < threshold;
+  }
+  return answer.probabilities[answer.choice] < threshold;
+}

package/src/extensions/jev/skills/gated-write.md

@@ -0,0 +1,25 @@
+---
+description: "Gate a channel wake, approval, or review comment on Jev by calling decide, then continuing or holding."
+---
+
+# Gating a GitHub write on Jev
+
+Channel hooks can call `decide` before they start a model turn. Return
+`null` when the answer misses the bar; return `{ auth }` or `{ task }`
+when the agent should run.
+
+This extension's `evaluate` tool does not write to GitHub. When an
+approval or a review comment must follow the judgment, add a server tool
+in this project that calls `decide`, then writes or holds in the same
+`execute`.
+
+```ts
+import { above, decide, needsHuman } from "@cursor/bdk/extensions/jev";
+```
+
+Hold when `needsHuman` is true or `above` misses the bar. Very low and
+low risk can approve; medium and above hold. A finding comment posts
+only when the draft is a real bug. Calibrate those numbers on labeled
+pull requests from this workflow. Pass the pull request title, summary,
+and draft, not the checkout. Return without writing when
+`ctx.session.purpose === "eval"`.

package/src/extensions/jev/skills/questions.md

@@ -0,0 +1,33 @@
+---
+description: "Ask Jev for a risk tier, a finding judgment, or a score when you are not about to approve or comment."
+---
+
+# Asking Jev
+
+Call this extension's `evaluate` tool when you need a typed judgment over
+a pull request and you are not about to approve or comment. It returns
+answers only. The tool appears as `<namespace>__evaluate`.
+
+## Questions
+
+Pass `state` (the pull request title, a short summary of the diff, and
+any draft finding) and one or more questions. Each question has an `id`,
+a `type`, and `instructions`.
+
+- `boolean`: probability of true between 0 and 1
+- `choice`: `criteria` maps each option name to a short description; the
+  answer names one option
+- `score`: `criteria` from lowest to highest; the answer is a number
+  along that rubric
+
+`instructions` can set the decision, who the review is for, and which
+evidence to weigh. Pass the pull request summary, not the checkout.
+
+## Reading answers
+
+A boolean answer has `probability`. A choice has `choice` and
+`probabilities`. A score has `score` and `probabilities`.
+
+When an approval or a review comment must follow the judgment, use a
+project tool that calls `decide` itself. Do not treat these answers as
+a GitHub write.

package/src/extensions/jev/tools/evaluate.ts

@@ -0,0 +1,82 @@
+import { z } from "zod";
+import { defineTool } from "../../../tools.js";
+import type { ToolDefinition } from "../../../types.js";
+import extension from "../extension.js";
+import { evaluate, type QuestionMap } from "../lib/evaluate.js";
+
+const booleanQuestion = z.object({
+  id: z.string().min(1),
+  type: z.literal("boolean"),
+  instructions: z.string().min(1),
+});
+
+const choiceQuestion = z.object({
+  id: z.string().min(1),
+  type: z.literal("choice"),
+  instructions: z.string().min(1),
+  criteria: z.record(z.string()),
+});
+
+const scoreQuestion = z.object({
+  id: z.string().min(1),
+  type: z.literal("score"),
+  instructions: z.string().min(1),
+  criteria: z.array(z.string()).min(2),
+});
+
+const question = z.discriminatedUnion("type", [booleanQuestion, choiceQuestion, scoreQuestion]);
+
+function toQuestionMap(questions: z.infer<typeof question>[]): QuestionMap {
+  const map: QuestionMap = {};
+  for (const item of questions) {
+    if (item.type === "boolean") {
+      map[item.id] = {
+        type: "boolean",
+        instructions: item.instructions,
+      };
+    } else if (item.type === "choice") {
+      map[item.id] = {
+        type: "choice",
+        instructions: item.instructions,
+        criteria: item.criteria,
+      };
+    } else {
+      map[item.id] = {
+        type: "score",
+        instructions: item.instructions,
+        criteria: item.criteria,
+      };
+    }
+  }
+  return map;
+}
+
+const evaluateTool: ToolDefinition = defineTool({
+  description:
+    "Ask Jev for typed boolean, choice, or score answers over one state. Returns answers only; it does not post, assign, or merge.",
+  effect: "read",
+  inputSchema: z
+    .object({
+      state: z
+        .union([z.string(), z.array(z.unknown()), z.record(z.unknown())])
+        .describe("Ticket, pull request, or other state Jev should judge. Not repo source."),
+      questions: z
+        .array(question)
+        .min(1)
+        .describe("One or more typed questions over the same state."),
+    })
+    .refine(
+      input => new Set(input.questions.map(item => item.id)).size === input.questions.length,
+      "questions must not repeat an id",
+    ),
+  async execute(input) {
+    return evaluate({
+      state: input.state,
+      questions: toQuestionMap(input.questions),
+      apiKey: extension.config.apiKey,
+      model: extension.config.model,
+    });
+  },
+});
+
+export default evaluateTool;