@orangekame3/pi-qcaleval 0.0.2-beta.5 → 0.0.2-beta.9

package/extensions/qcaleval.ts

@@ -20,6 +20,10 @@ import { Type, type Static } from "typebox";
 /** Figures per evaluation; a 16k-token deployment holds a few plots comfortably. */
 export const MAX_IMAGES = 4;
 
+/** Labeled example plots shown before the query, and the figures they may hold in total. */
+export const MAX_EXAMPLES = 4;
+export const MAX_EXAMPLE_IMAGES = 4;
+
 /** Model ids of models fine-tuned for calibration plots; preferred when configured. */
 export const CALIBRATION_MODEL_PATTERN = /ising-calibration/i;
 
@@ -113,39 +117,111 @@ export const SYSTEM_PROMPT = [
   "Base every statement on the plot, the reference, and the given context only.",
 ].join("\n");
 
+/** One labeled example: its figure(s) followed by the expert's diagnosis. */
+export interface LabeledExample {
+  label: string;
+  images: ReadonlyArray<{ data: string; mimeType: string }>;
+}
+
+export type UserContentBlock =
+  | { type: "text"; text: string }
+  | { type: "image"; data: string; mimeType: string };
+
 /**
- * Build the single user turn sent to the evaluation model. The host's
- * reference for the task, when given, comes first so the context and the plot
- * are read against it. With several figures, the model is told how many follow
- * so it reads them in the order the context describes (expected reference
- * first, measured result last).
+ * The single user turn sent to the evaluation model, as the QCalEval papers
+ * lay it out: the reference for the experiment, then each labeled example as
+ * its figure followed by its diagnosis, then the context and question, then
+ * the figure to judge. Everything is one ordered content array; images are
+ * native blocks, never placeholders in the text.
+ *
+ * `prompt` is the same turn as text, with `[figure]` where an image was, for
+ * the record of what the model saw.
  */
-export function buildUserPrompt(
+export function buildUserContent(
   context: string,
   question: string | undefined,
-  imageCount = 1,
+  images: ReadonlyArray<{ data: string; mimeType: string }>,
   knowledge?: string,
-): string {
-  const lines: string[] = [];
+  examples: ReadonlyArray<LabeledExample> = [],
+): { content: UserContentBlock[]; prompt: string; examplesUsed: number } {
+  const content: UserContentBlock[] = [];
+  const shown = fitExamples(examples);
+  const text = (lines: string[]) => {
+    content.push({ type: "text", text: lines.join("\n") });
+  };
+  const image = (figure: { data: string; mimeType: string }) => {
+    content.push({ type: "image", data: figure.data, mimeType: figure.mimeType });
+  };
+
+  const opening: string[] = [];
   if (knowledge?.trim()) {
-    lines.push(
-      "Reference for this experiment (expected result, review questions, failure patterns):",
-      knowledge.trim(),
-      "",
+    opening.push("Reference for this experiment (expected plot, outcomes, parameters):", knowledge.trim(), "");
+  }
+  if (shown.length > 0) {
+    opening.push(
+      `${shown.length} labeled example${shown.length === 1 ? "" : "s"} of this experiment follow, each figure followed by the diagnosis an expert validated. Use them as the standard for the figure to judge, which comes last.`,
     );
   }
-  lines.push("Experiment context:", context.trim() || "(none)");
-  if (imageCount > 1) {
-    lines.push(
+  if (opening.length > 0) text(opening);
+
+  shown.forEach((example, index) => {
+    text(["", `Example ${index + 1} of ${shown.length}:`]);
+    for (const figure of example.images) image(figure);
+    text([`Diagnosis of example ${index + 1}:`, example.label.trim()]);
+  });
+
+  const closing: string[] = [];
+  if (shown.length > 0) closing.push("", "Now the figure to judge, evaluated independently of the examples.");
+  closing.push("Experiment context:", context.trim() || "(none)");
+  if (images.length > 1) {
+    closing.push(
       "",
-      `${imageCount} figures follow, in the order the context describes; the last one is the measured result to judge.`,
+      `${images.length} figures follow, in the order the context describes; the last one is the measured result to judge.`,
     );
   }
-  if (question?.trim()) lines.push("", "Specific question:", question.trim());
+  if (question?.trim()) closing.push("", "Specific question:", question.trim());
   // Repeated here because a long reference ahead of the plot pulls the model
   // towards prose; the last instruction it reads is the output format.
-  lines.push("", "Answer with the JSON object described in the system prompt, and nothing else.");
-  return lines.join("\n");
+  closing.push("", "Answer with the JSON object described in the system prompt, and nothing else.");
+  text(closing);
+  for (const figure of images) image(figure);
+
+  const prompt = content
+    .map((block) => (block.type === "text" ? block.text : "[figure]"))
+    .join("\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+  return { content, prompt, examplesUsed: shown.length };
+}
+
+/** Keep examples in order within the example and figure budgets; later ones are dropped. */
+export function fitExamples(examples: ReadonlyArray<LabeledExample>): LabeledExample[] {
+  const kept: LabeledExample[] = [];
+  let figures = 0;
+  for (const example of examples) {
+    if (kept.length >= MAX_EXAMPLES) break;
+    if (!example.label?.trim() || !example.images?.length) continue;
+    if (figures + example.images.length > MAX_EXAMPLE_IMAGES) break;
+    figures += example.images.length;
+    kept.push(example);
+  }
+  return kept;
+}
+
+/** The user turn as text only (no examples), kept for callers that render the prompt. */
+export function buildUserPrompt(
+  context: string,
+  question: string | undefined,
+  imageCount = 1,
+  knowledge?: string,
+): string {
+  const placeholder = { data: "", mimeType: "image/png" };
+  return buildUserContent(context, question, Array(imageCount).fill(placeholder), knowledge)
+    .content.filter((block) => block.type === "text")
+    .map((block) => (block as { text: string }).text)
+    .join("\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
 }
 
 /** Pull the text blocks of the assistant answer, dropping thinking. */
@@ -206,6 +282,11 @@ const imageSchema = Type.Object({
   mimeType: Type.String({ description: "image/png or image/jpeg" }),
 });
 
+const exampleSchema = Type.Object({
+  label: Type.String({ description: "The expert's diagnosis of the example figure(s)" }),
+  images: Type.Array(imageSchema, { minItems: 1 }),
+});
+
 const parameters = Type.Object({
   context: Type.String({
     description:
@@ -240,6 +321,19 @@ const parameters = Type.Object({
         "How many of the newest figures the host should supply when `images` is omitted (default 1). Use 2 when an expected reference figure precedes the measured one, and describe their order in `context`.",
     }),
   ),
+  use_examples: Type.Optional(
+    Type.Boolean({
+      description:
+        "Set true only when the user asked to judge against labeled reference examples: the host then supplies its expert-labeled example plots for `task_name` (in-context examples). Default false.",
+    }),
+  ),
+  examples: Type.Optional(
+    Type.Array(exampleSchema, {
+      maxItems: MAX_EXAMPLES,
+      description:
+        "Normally omitted: the host supplies labeled examples when `use_examples` is true. Each is one or more real figures followed by the expert's diagnosis.",
+    }),
+  ),
 });
 
 /** Stored on the tool result; the host reads `diagnosis` without reparsing text. */
@@ -251,8 +345,12 @@ export interface EvaluationDetails {
   diagnosis?: Record<string, unknown> | null;
   /** The model's answer when it was not the requested JSON. */
   raw?: string;
-  /** The user turn the model was sent (reference, context, question), so the host can show what it saw. */
+  /** The user turn the model was sent (reference, examples, context, question), so the host can show what it saw. */
   prompt?: string;
+  /** How many labeled examples were shown before the figure. */
+  examples_used?: number;
+  /** The system prompt the model was sent, so the host can show the whole request. */
+  system_prompt?: string;
 }
 
 function failure(spec: ModelSpec | undefined, text: string): AgentToolResult<EvaluationDetails> {
@@ -274,6 +372,7 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
       "Use qcal_evaluate for every calibration figure you need to judge, instead of judging it yourself.",
       "Pass the task name, target, and expected result as context so the diagnosis is specific.",
       "Pass task_name when you know it: the host then supplies its reference knowledge for that task.",
+      "Set use_examples true only when the user asked for a judgement against labeled reference examples.",
       "Treat the returned diagnosis as the evaluation of record; add your own reasoning only on top of it.",
     ],
     parameters,
@@ -297,7 +396,13 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
         return failure(spec, "no plot was given. Pass the figure in `images` or fetch it first.");
       }
 
-      const prompt = buildUserPrompt(params.context, params.question, images.length, params.knowledge);
+      const { content, prompt, examplesUsed } = buildUserContent(
+        params.context,
+        params.question,
+        images.map((image) => ({ data: image.data, mimeType: image.mimeType })),
+        params.knowledge,
+        params.examples ?? [],
+      );
       const message = await ctx.modelRegistry.complete(
         model,
         {
@@ -305,10 +410,11 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
           messages: [
             {
               role: "user",
-              content: [
-                { type: "text", text: prompt },
-                ...images,
-              ],
+              content: content.map((block) =>
+                block.type === "text"
+                  ? { type: "text" as const, text: block.text }
+                  : { type: "image" as const, data: block.data, mimeType: block.mimeType },
+              ),
               timestamp: Date.now(),
             },
           ],
@@ -328,6 +434,8 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
           diagnosis: diagnosis ?? null,
           ...(diagnosis ? {} : { raw: text }),
           prompt,
+          system_prompt: SYSTEM_PROMPT,
+          ...(examplesUsed ? { examples_used: examplesUsed } : {}),
         },
         ...(message.usage ? { usage: message.usage } : {}),
       };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orangekame3/pi-qcaleval",
-  "version": "0.0.2-beta.5",
+  "version": "0.0.2-beta.9",
   "description": "pi extension that sends calibration plots to a calibration-evaluation VLM such as NVIDIA Ising-Calibration and returns a structured diagnosis",
   "license": "Apache-2.0",
   "repository": {

package/skills/qcaleval/SKILL.md

@@ -27,6 +27,14 @@ Ising-Calibration by default) and returns a JSON diagnosis with
   in `context` that the first figure is the reference and the last the
   measurement. The host then supplies both; the default is the newest figure only.
 
+## Labeled examples (in-context)
+
+- When the user asks to judge a plot against the labeled reference examples
+  (in-context examples, few-shot), pass `use_examples: true` with `task_name`.
+  The host supplies its expert-labeled example plots for that task, and the
+  model sees each example figure followed by its diagnosis before the figure
+  to judge, as in the QCalEval papers. Leave it off otherwise.
+
 ## How to use the result
 
 - Treat the diagnosis as the evaluation of record; reason on top of it instead