@orangekame3/pi-qcaleval 0.0.1 → 0.0.2-beta.10

package/README.md

@@ -13,23 +13,43 @@ works from the pi CLI and from the QDash Copilot runtime alike.
 
 ## Tool
 
-`qcal_evaluate(context, question?, images?, max_images?)` returns a JSON object with
+`qcal_evaluate(context, question?, task_name?, knowledge?, images?, max_images?)` returns a JSON object with
 `technical_description`, `experimental_conclusion`, `experimental_significance`,
 `fit_quality`, `parameter_extraction`, and `success`. The parsed object is also
 stored in the tool result's `details.diagnosis`; when the model did not answer
-with JSON, `details.raw` holds its text instead.
+with JSON, or not with all of it, `details.raw` holds its text and
+`details.missing_keys` the keys it lacks.
+
+On a chat-completions API (vLLM, SGLang, llama.cpp) the request carries a
+`response_format` with the diagnosis schema, so decoding is constrained to the
+six keys; a server that refuses it is asked plainly. An answer that still
+lacks keys is asked for once more and the more complete one kept. The details
+record `attempts` (1 or 2), `structured` (whether decoding was constrained),
+and `stop_reason`.
 
 `images` are base64 image blocks. A host may fill them in from the most recent
 figures of the conversation, so the planner does not have to pass image bytes;
 `max_images` (default 1, up to 4) says how many of the newest figures to supply,
 for example 2 when an expected reference figure precedes the measured result.
 
+`knowledge` is the host's reference for the experiment: what a good result
+looks like, the questions a reviewer asks, and the common failure patterns. A
+host that keeps such knowledge per task fills it in from `task_name`, so the
+planner only names the task; the text is sent before the context. The exact
+user turn the model received (reference, context, question) is stored in
+`details.prompt`, so a host can show what the model saw.
+
 ## Configuration
 
-| Variable | Purpose |
-|----------|---------|
-| `QCALEVAL_MODEL` | `provider/modelId` of the evaluation model in pi's `models.json` (default: `vllm/nvidia/Ising-Calibration-1.5-31B-NVFP4`) |
-| `QCALEVAL_MAX_TOKENS` | Output budget for one evaluation (default: `2048`) |
+No setting is required. The evaluation model is resolved in this order:
+
+1. `QCALEVAL_MODEL` (`provider/modelId`), an explicit override for pi CLI use;
+2. a configured calibration VLM, recognised by its id (`Ising-Calibration`);
+3. the conversation's own model, when it accepts images. The result is then
+   marked `fallback: true` in `details`, so a host without an Ising deployment
+   still gets a diagnosis from the vision model it already runs.
+
+The output budget follows the model's `maxTokens` (at most 4096).
 
 The provider must be declared in pi's `models.json` with its base URL and
 credential, for example:

package/extensions/qcaleval.ts

@@ -2,13 +2,14 @@
  * pi-qcaleval: evaluate calibration plots with a dedicated calibration VLM.
  *
  * The extension has no QDash dependency. It takes a plot (plus the experiment
- * context the planner already knows) and asks the evaluation model, which is
+ * context the planner already knows, and the host's reference knowledge for
+ * the task when the host supplies one) and asks the evaluation model, which is
  * configured separately from the conversation's model, for a diagnosis. The
  * planner decides what to do with it; whether a calibration loop continues is
  * the host's policy, not this tool's.
  */
 
-import type { Api, AssistantMessage, ImageContent, Model } from "@earendil-works/pi-ai";
+import type { Api, AssistantMessage, ImageContent, Model, Usage } from "@earendil-works/pi-ai";
 import type {
   AgentToolResult,
   ExtensionAPI,
@@ -16,20 +17,29 @@ import type {
 } from "@earendil-works/pi-coding-agent";
 import { Type, type Static } from "typebox";
 
-/** `provider/modelId`; the model id itself may contain slashes (nvidia/...). */
-export const DEFAULT_MODEL = "vllm/nvidia/Ising-Calibration-1.5-31B-NVFP4";
-
 /** Figures per evaluation; a 16k-token deployment holds a few plots comfortably. */
 export const MAX_IMAGES = 4;
 
+/** Labeled example plots shown before the query, and the figures they may hold in total. */
+export const MAX_EXAMPLES = 4;
+export const MAX_EXAMPLE_IMAGES = 4;
+
+/** Model ids of models fine-tuned for calibration plots; preferred when configured. */
+export const CALIBRATION_MODEL_PATTERN = /ising-calibration/i;
+
+/** Output budget when the model declares none; a diagnosis is a few hundred tokens. */
+const DEFAULT_MAX_TOKENS = 2048;
+const MAX_OUTPUT_TOKENS = 4096;
+
 export interface ModelSpec {
   provider: string;
   modelId: string;
 }
 
-/** Parse `provider/modelId`, splitting at the first slash only. */
-export function parseModelSpec(value: string | undefined): ModelSpec {
-  const spec = (value ?? "").trim() || DEFAULT_MODEL;
+/** Parse an explicit `provider/modelId` override; the model id may contain slashes (nvidia/...). */
+export function parseModelSpec(value: string | undefined): ModelSpec | undefined {
+  const spec = (value ?? "").trim();
+  if (!spec) return undefined;
   const slash = spec.indexOf("/");
   if (slash <= 0 || slash === spec.length - 1) {
     throw new Error(`QCALEVAL_MODEL must be provider/modelId, got "${spec}"`);
@@ -37,6 +47,54 @@ export function parseModelSpec(value: string | undefined): ModelSpec {
   return { provider: spec.slice(0, slash), modelId: spec.slice(slash + 1) };
 }
 
+/** The slice of pi's model registry the resolution needs. */
+export interface ModelLookup {
+  find(provider: string, modelId: string): Model<Api> | undefined;
+  getAvailable(): Model<Api>[];
+}
+
+export type EvaluationModel =
+  | { model: Model<Api>; fallback: false }
+  | { model: Model<Api>; fallback: true }
+  | { model?: undefined; error: string };
+
+/**
+ * Pick the model that judges the plot, without any host-specific setting:
+ *
+ * 1. `QCALEVAL_MODEL` (`provider/modelId`), an explicit override for the pi CLI;
+ * 2. a configured calibration VLM, recognised by its id (Ising-Calibration);
+ * 3. the conversation's own model, when it accepts images.
+ *
+ * A host that declares the calibration model in its models.json gets it
+ * everywhere; one that does not (no Ising deployment) still gets a diagnosis
+ * from the vision model it already runs, marked as a fallback.
+ */
+export function resolveEvaluationModel(
+  registry: ModelLookup,
+  current: Model<Api> | undefined,
+  override: string | undefined,
+): EvaluationModel {
+  const spec = parseModelSpec(override);
+  if (spec) {
+    const model = registry.find(spec.provider, spec.modelId);
+    return model
+      ? { model, fallback: false }
+      : { error: `model ${spec.provider}/${spec.modelId} is not configured (QCALEVAL_MODEL)` };
+  }
+  const calibration = registry.getAvailable().find((model) => CALIBRATION_MODEL_PATTERN.test(model.id));
+  if (calibration) return { model: calibration, fallback: false };
+  if (current?.input.includes("image")) return { model: current, fallback: true };
+  return {
+    error:
+      "no evaluation model: declare a calibration VLM (Ising-Calibration) in models.json, or run with a vision model",
+  };
+}
+
+/** Output budget for one evaluation, within what the model allows. */
+export function outputBudget(model: Pick<Model<Api>, "maxTokens">): number {
+  return model.maxTokens > 0 ? Math.min(model.maxTokens, MAX_OUTPUT_TOKENS) : DEFAULT_MAX_TOKENS;
+}
+
 /** Categories the Ising-Calibration model card describes; asked for in this order. */
 export const DIAGNOSIS_SECTIONS = [
   "technical_description",
@@ -47,6 +105,50 @@ export const DIAGNOSIS_SECTIONS = [
   "success",
 ] as const;
 
+/**
+ * The diagnosis as a JSON schema: the six keys, all required. Sent as the
+ * response format to OpenAI-compatible servers that constrain decoding to a
+ * schema (vLLM, SGLang, llama.cpp). Asked for JSON in the prompt alone, the
+ * calibration model now and then closes the object after the first key or
+ * nests quotes that break it; constrained decoding rules both out.
+ */
+export const DIAGNOSIS_SCHEMA = {
+  type: "object",
+  properties: {
+    technical_description: { type: "string" },
+    experimental_conclusion: { type: "string" },
+    experimental_significance: { type: "string" },
+    fit_quality: { type: "string" },
+    parameter_extraction: {
+      anyOf: [{ type: "object", additionalProperties: true }, { type: "string" }, { type: "null" }],
+    },
+    success: { type: "boolean" },
+  },
+  required: [...DIAGNOSIS_SECTIONS],
+  additionalProperties: false,
+} as const;
+
+/** `response_format` of a chat completion constrained to the diagnosis schema. */
+export const RESPONSE_FORMAT = {
+  type: "json_schema",
+  json_schema: { name: "calibration_diagnosis", schema: DIAGNOSIS_SCHEMA },
+} as const;
+
+/**
+ * Whether the model's API takes `response_format` in the request body. Pi
+ * merges `samplingParams` into the body of chat completions only; the
+ * responses API spells the format differently and other APIs ignore it.
+ */
+export function canConstrain(model: Pick<Model<Api>, "api">): boolean {
+  return model.api === "openai-completions";
+}
+
+/** The diagnosis keys an answer lacks; all six when there is no diagnosis. */
+export function missingKeys(diagnosis: Record<string, unknown> | undefined): string[] {
+  if (!diagnosis) return [...DIAGNOSIS_SECTIONS];
+  return DIAGNOSIS_SECTIONS.filter((key) => !(key in diagnosis));
+}
+
 export const SYSTEM_PROMPT = [
   "You evaluate superconducting qubit calibration experiment plots.",
   "Answer with one JSON object and nothing else, with these keys:",
@@ -56,28 +158,114 @@ export const SYSTEM_PROMPT = [
   '  "fit_quality": "good" | "marginal" | "poor" and why;',
   '  "parameter_extraction": extracted values with units, or null;',
   '  "success": true | false, whether the experiment achieved its goal.',
-  "Base every statement on the plot and the given context only.",
+  "Base every statement on the plot, the reference, and the given context only.",
 ].join("\n");
 
+/** One labeled example: its figure(s) followed by the expert's diagnosis. */
+export interface LabeledExample {
+  label: string;
+  images: ReadonlyArray<{ data: string; mimeType: string }>;
+}
+
+export type UserContentBlock =
+  | { type: "text"; text: string }
+  | { type: "image"; data: string; mimeType: string };
+
 /**
- * Build the single user turn sent to the evaluation model. With several
- * figures, the model is told how many follow so it reads them in the order
- * the context describes (expected reference first, measured result last).
+ * The single user turn sent to the evaluation model, as the QCalEval papers
+ * lay it out: the reference for the experiment, then each labeled example as
+ * its figure followed by its diagnosis, then the context and question, then
+ * the figure to judge. Everything is one ordered content array; images are
+ * native blocks, never placeholders in the text.
+ *
+ * `prompt` is the same turn as text, with `[figure]` where an image was, for
+ * the record of what the model saw.
  */
-export function buildUserPrompt(
+export function buildUserContent(
   context: string,
   question: string | undefined,
-  imageCount = 1,
-): string {
-  const lines = ["Experiment context:", context.trim() || "(none)"];
-  if (imageCount > 1) {
-    lines.push(
+  images: ReadonlyArray<{ data: string; mimeType: string }>,
+  knowledge?: string,
+  examples: ReadonlyArray<LabeledExample> = [],
+): { content: UserContentBlock[]; prompt: string; examplesUsed: number } {
+  const content: UserContentBlock[] = [];
+  const shown = fitExamples(examples);
+  const text = (lines: string[]) => {
+    content.push({ type: "text", text: lines.join("\n") });
+  };
+  const image = (figure: { data: string; mimeType: string }) => {
+    content.push({ type: "image", data: figure.data, mimeType: figure.mimeType });
+  };
+
+  const opening: string[] = [];
+  if (knowledge?.trim()) {
+    opening.push("Reference for this experiment (expected plot, outcomes, parameters):", knowledge.trim(), "");
+  }
+  if (shown.length > 0) {
+    opening.push(
+      `${shown.length} labeled example${shown.length === 1 ? "" : "s"} of this experiment follow, each figure followed by the diagnosis an expert validated. Use them as the standard for the figure to judge, which comes last.`,
+    );
+  }
+  if (opening.length > 0) text(opening);
+
+  shown.forEach((example, index) => {
+    text(["", `Example ${index + 1} of ${shown.length}:`]);
+    for (const figure of example.images) image(figure);
+    text([`Diagnosis of example ${index + 1}:`, example.label.trim()]);
+  });
+
+  const closing: string[] = [];
+  if (shown.length > 0) closing.push("", "Now the figure to judge, evaluated independently of the examples.");
+  closing.push("Experiment context:", context.trim() || "(none)");
+  if (images.length > 1) {
+    closing.push(
       "",
-      `${imageCount} figures follow, in the order the context describes; the last one is the measured result to judge.`,
+      `${images.length} figures follow, in the order the context describes; the last one is the measured result to judge.`,
     );
   }
-  if (question?.trim()) lines.push("", "Specific question:", question.trim());
-  return lines.join("\n");
+  if (question?.trim()) closing.push("", "Specific question:", question.trim());
+  // Repeated here because a long reference ahead of the plot pulls the model
+  // towards prose; the last instruction it reads is the output format.
+  closing.push("", "Answer with the JSON object described in the system prompt, and nothing else.");
+  text(closing);
+  for (const figure of images) image(figure);
+
+  const prompt = content
+    .map((block) => (block.type === "text" ? block.text : "[figure]"))
+    .join("\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+  return { content, prompt, examplesUsed: shown.length };
+}
+
+/** Keep examples in order within the example and figure budgets; later ones are dropped. */
+export function fitExamples(examples: ReadonlyArray<LabeledExample>): LabeledExample[] {
+  const kept: LabeledExample[] = [];
+  let figures = 0;
+  for (const example of examples) {
+    if (kept.length >= MAX_EXAMPLES) break;
+    if (!example.label?.trim() || !example.images?.length) continue;
+    if (figures + example.images.length > MAX_EXAMPLE_IMAGES) break;
+    figures += example.images.length;
+    kept.push(example);
+  }
+  return kept;
+}
+
+/** The user turn as text only (no examples), kept for callers that render the prompt. */
+export function buildUserPrompt(
+  context: string,
+  question: string | undefined,
+  imageCount = 1,
+  knowledge?: string,
+): string {
+  const placeholder = { data: "", mimeType: "image/png" };
+  return buildUserContent(context, question, Array(imageCount).fill(placeholder), knowledge)
+    .content.filter((block) => block.type === "text")
+    .map((block) => (block as { text: string }).text)
+    .join("\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
 }
 
 /** Pull the text blocks of the assistant answer, dropping thinking. */
@@ -133,11 +321,99 @@ function parseObject(json: string): Record<string, unknown> | undefined {
   return undefined;
 }
 
+/** One answer of the evaluation model, read for the record. */
+export interface Answer {
+  text: string;
+  diagnosis?: Record<string, unknown>;
+  /** Diagnosis keys the answer lacks; empty when it is complete. */
+  missing: string[];
+  /** Model calls made for this answer: 1, or 2 after an incomplete or refused first one. */
+  attempts: number;
+  /** Whether decoding was constrained to the diagnosis schema. */
+  structured: boolean;
+  stopReason: AssistantMessage["stopReason"];
+  usage?: Usage;
+}
+
+function readAnswer(message: AssistantMessage, structured: boolean): Answer {
+  const text = answerText(message);
+  const diagnosis = parseDiagnosis(text);
+  return {
+    text,
+    diagnosis,
+    missing: missingKeys(diagnosis),
+    attempts: 1,
+    structured,
+    stopReason: message.stopReason,
+    usage: message.usage,
+  };
+}
+
+/** Token usage of two calls together, so a retried answer is billed as both. */
+export function addUsage(a: Usage | undefined, b: Usage | undefined): Usage | undefined {
+  if (!a || !b) return a ?? b;
+  const sum = (x: number | undefined, y: number | undefined) => (x ?? 0) + (y ?? 0);
+  return {
+    ...a,
+    input: sum(a.input, b.input),
+    output: sum(a.output, b.output),
+    cacheRead: sum(a.cacheRead, b.cacheRead),
+    cacheWrite: sum(a.cacheWrite, b.cacheWrite),
+    ...(a.reasoning === undefined && b.reasoning === undefined
+      ? {}
+      : { reasoning: sum(a.reasoning, b.reasoning) }),
+    totalTokens: sum(a.totalTokens, b.totalTokens),
+    cost: {
+      input: sum(a.cost?.input, b.cost?.input),
+      output: sum(a.cost?.output, b.cost?.output),
+      cacheRead: sum(a.cost?.cacheRead, b.cost?.cacheRead),
+      cacheWrite: sum(a.cost?.cacheWrite, b.cost?.cacheWrite),
+      total: sum(a.cost?.total, b.cost?.total),
+    },
+  };
+}
+
+/**
+ * Ask the model for the diagnosis, in at most two calls. The first call is
+ * constrained to the schema when the API allows it; a server that refuses the
+ * response format (an older vLLM, a gateway) gets the plain request instead.
+ * An answer that lacks diagnosis keys is asked for once more, since the model
+ * usually completes the object on a second sample, and the more complete of
+ * the two answers is kept. Usage counts every call made.
+ */
+export async function askForDiagnosis(
+  call: (structured: boolean) => Promise<AssistantMessage>,
+  constrain: boolean,
+): Promise<Answer | { error: string }> {
+  let structured = constrain;
+  let first = await call(structured);
+  let attempts = 1;
+  let usage: Usage | undefined = first.usage;
+  if (first.stopReason === "error" && structured) {
+    structured = false;
+    first = await call(structured);
+    attempts += 1;
+    usage = addUsage(usage, first.usage);
+  }
+  if (first.stopReason === "error") return { error: first.errorMessage ?? "model error" };
+  const answer = { ...readAnswer(first, structured), attempts, usage };
+  if (answer.missing.length === 0 || attempts === 2) return answer;
+  const second = await call(structured);
+  const retried = second.stopReason === "error" ? undefined : readAnswer(second, structured);
+  const kept = retried && retried.missing.length <= answer.missing.length ? retried : answer;
+  return { ...kept, attempts: 2, usage: addUsage(first.usage, second.usage) };
+}
+
 const imageSchema = Type.Object({
   data: Type.String({ description: "Base64 image bytes (no data: prefix)" }),
   mimeType: Type.String({ description: "image/png or image/jpeg" }),
 });
 
+const exampleSchema = Type.Object({
+  label: Type.String({ description: "The expert's diagnosis of the example figure(s)" }),
+  images: Type.Array(imageSchema, { minItems: 1 }),
+});
+
 const parameters = Type.Object({
   context: Type.String({
     description:
@@ -146,6 +422,18 @@ const parameters = Type.Object({
   question: Type.Optional(
     Type.String({ description: "A specific question to answer about the plot" }),
   ),
+  task_name: Type.Optional(
+    Type.String({
+      description:
+        "The calibration task's name as the host knows it (e.g. RandomizedBenchmarking). Lets the host supply its reference knowledge for the experiment.",
+    }),
+  ),
+  knowledge: Type.Optional(
+    Type.String({
+      description:
+        "Normally omitted: the host supplies the reference for `task_name` (expected result, review questions, failure patterns). Fill only to override it with text you were given.",
+    }),
+  ),
   images: Type.Optional(
     Type.Array(imageSchema, {
       description:
@@ -160,18 +448,52 @@ const parameters = Type.Object({
         "How many of the newest figures the host should supply when `images` is omitted (default 1). Use 2 when an expected reference figure precedes the measured one, and describe their order in `context`.",
     }),
   ),
+  use_examples: Type.Optional(
+    Type.Boolean({
+      description:
+        "Set true only when the user asked to judge against labeled reference examples: the host then supplies its expert-labeled example plots for `task_name` (in-context examples). Default false.",
+    }),
+  ),
+  examples: Type.Optional(
+    Type.Array(exampleSchema, {
+      maxItems: MAX_EXAMPLES,
+      description:
+        "Normally omitted: the host supplies labeled examples when `use_examples` is true. Each is one or more real figures followed by the expert's diagnosis.",
+    }),
+  ),
 });
 
 /** Stored on the tool result; the host reads `diagnosis` without reparsing text. */
 export interface EvaluationDetails {
-  model: ModelSpec;
+  /** The model that judged the plot; absent when none could be resolved. */
+  model?: ModelSpec;
+  /** True when no calibration VLM was configured and the conversation's model judged instead. */
+  fallback?: boolean;
   diagnosis?: Record<string, unknown> | null;
-  /** The model's answer when it was not the requested JSON. */
+  /** The model's answer when it was not the requested JSON, or not all of it. */
   raw?: string;
+  /** Diagnosis keys the parsed answer lacks, when it is incomplete. */
+  missing_keys?: string[];
+  /** Model calls made: 1, or 2 after an incomplete or refused first answer. */
+  attempts?: number;
+  /** Whether decoding was constrained to the diagnosis schema. */
+  structured?: boolean;
+  /** Why the model stopped, as pi reports it. */
+  stop_reason?: string;
+  /** The user turn the model was sent (reference, examples, context, question), so the host can show what it saw. */
+  prompt?: string;
+  /** How many labeled examples were shown before the figure. */
+  examples_used?: number;
+  /** The system prompt the model was sent, so the host can show the whole request. */
+  system_prompt?: string;
 }
 
-function failure(spec: ModelSpec, text: string): AgentToolResult<EvaluationDetails> {
-  return { content: [{ type: "text", text: `qcal_evaluate: ${text}` }], details: { model: spec }, isError: true };
+function failure(spec: ModelSpec | undefined, text: string): AgentToolResult<EvaluationDetails> {
+  return {
+    content: [{ type: "text", text: `qcal_evaluate: ${text}` }],
+    details: spec ? { model: spec } : {},
+    isError: true,
+  };
 }
 
 export default function qcalevalExtension(pi: ExtensionAPI) {
@@ -184,6 +506,8 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
     promptGuidelines: [
       "Use qcal_evaluate for every calibration figure you need to judge, instead of judging it yourself.",
       "Pass the task name, target, and expected result as context so the diagnosis is specific.",
+      "Pass task_name when you know it: the host then supplies its reference knowledge for that task.",
+      "Set use_examples true only when the user asked for a judgement against labeled reference examples.",
       "Treat the returned diagnosis as the evaluation of record; add your own reasoning only on top of it.",
     ],
     parameters,
@@ -194,11 +518,10 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
       _onUpdate: unknown,
       ctx: ExtensionToolContext,
     ): Promise<AgentToolResult<EvaluationDetails>> {
-      const spec = parseModelSpec(process.env.QCALEVAL_MODEL);
-      const model: Model<Api> | undefined = ctx.modelRegistry.find(spec.provider, spec.modelId);
-      if (!model) {
-        return failure(spec, `model ${spec.provider}/${spec.modelId} is not configured (QCALEVAL_MODEL)`);
-      }
+      const resolved = resolveEvaluationModel(ctx.modelRegistry, ctx.model, process.env.QCALEVAL_MODEL);
+      if (!resolved.model) return failure(undefined, resolved.error);
+      const { model, fallback } = resolved;
+      const spec: ModelSpec = { provider: model.provider, modelId: model.id };
       const images: ImageContent[] = (params.images ?? []).map((image) => ({
         type: "image",
         data: image.data,
@@ -208,39 +531,58 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
         return failure(spec, "no plot was given. Pass the figure in `images` or fetch it first.");
       }
 
-      const message = await ctx.modelRegistry.complete(
-        model,
-        {
-          systemPrompt: SYSTEM_PROMPT,
-          messages: [
+      const { content, prompt, examplesUsed } = buildUserContent(
+        params.context,
+        params.question,
+        images.map((image) => ({ data: image.data, mimeType: image.mimeType })),
+        params.knowledge,
+        params.examples ?? [],
+      );
+      const answer = await askForDiagnosis(
+        (structured) =>
+          ctx.modelRegistry.complete(
+            model,
             {
-              role: "user",
-              content: [
+              systemPrompt: SYSTEM_PROMPT,
+              messages: [
                 {
-                  type: "text",
-                  text: buildUserPrompt(params.context, params.question, images.length),
+                  role: "user",
+                  content: content.map((block) =>
+                    block.type === "text"
+                      ? { type: "text" as const, text: block.text }
+                      : { type: "image" as const, data: block.data, mimeType: block.mimeType },
+                  ),
+                  timestamp: Date.now(),
                 },
-                ...images,
               ],
-              timestamp: Date.now(),
             },
-          ],
-        },
-        { signal, maxTokens: Number(process.env.QCALEVAL_MAX_TOKENS ?? 2048) },
+            {
+              signal,
+              maxTokens: outputBudget(model),
+              ...(structured ? { samplingParams: { response_format: RESPONSE_FORMAT } } : {}),
+            },
+          ),
+        canConstrain(model),
       );
-      if (message.stopReason === "error") {
-        return failure(spec, message.errorMessage ?? "model error");
-      }
-      const text = answerText(message);
-      const diagnosis = parseDiagnosis(text);
+      if ("error" in answer) return failure(spec, answer.error);
+      const { text, diagnosis, missing } = answer;
+      const complete = diagnosis !== undefined && missing.length === 0;
       return {
         content: [{ type: "text", text: diagnosis ? JSON.stringify(diagnosis, null, 2) : text }],
         details: {
           model: spec,
+          ...(fallback ? { fallback: true } : {}),
           diagnosis: diagnosis ?? null,
-          ...(diagnosis ? {} : { raw: text }),
+          ...(complete ? {} : { raw: text }),
+          ...(diagnosis && !complete ? { missing_keys: missing } : {}),
+          attempts: answer.attempts,
+          structured: answer.structured,
+          stop_reason: answer.stopReason,
+          prompt,
+          system_prompt: SYSTEM_PROMPT,
+          ...(examplesUsed ? { examples_used: examplesUsed } : {}),
         },
-        ...(message.usage ? { usage: message.usage } : {}),
+        ...(answer.usage ? { usage: answer.usage } : {}),
       };
     },
   });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orangekame3/pi-qcaleval",
-  "version": "0.0.1",
+  "version": "0.0.2-beta.10",
   "description": "pi extension that sends calibration plots to a calibration-evaluation VLM such as NVIDIA Ising-Calibration and returns a structured diagnosis",
   "license": "Apache-2.0",
   "repository": {

package/skills/qcaleval/SKILL.md

@@ -18,11 +18,23 @@ Ising-Calibration by default) and returns a JSON diagnosis with
   the task name, the target (qubit or coupling), the swept parameter, and what a
   successful result looks like. Add `question` when a specific point matters
   (for example "is the chevron centred on the current frequency?").
+- Pass `task_name` whenever you know the task (for example `CheckRabi`): the
+  host then supplies its own reference knowledge for that experiment, which is
+  more reliable than a summary you write yourself. Keep `context` for what the
+  reference cannot know: the target, the swept range, and previous values.
 - When the conversation holds an expected reference figure followed by the
   measured result (a task review opens this way), pass `max_images: 2` and say
   in `context` that the first figure is the reference and the last the
   measurement. The host then supplies both; the default is the newest figure only.
 
+## Labeled examples (in-context)
+
+- When the user asks to judge a plot against the labeled reference examples
+  (in-context examples, few-shot), pass `use_examples: true` with `task_name`.
+  The host supplies its expert-labeled example plots for that task, and the
+  model sees each example figure followed by its diagnosis before the figure
+  to judge, as in the QCalEval papers. Leave it off otherwise.
+
 ## How to use the result
 
 - Treat the diagnosis as the evaluation of record; reason on top of it instead