@orangekame3/pi-qcaleval 0.0.2-beta.3 → 0.0.2-beta.5

package/README.md

@@ -13,7 +13,7 @@ works from the pi CLI and from the QDash Copilot runtime alike.
 
 ## Tool
 
-`qcal_evaluate(context, question?, images?, max_images?)` returns a JSON object with
+`qcal_evaluate(context, question?, task_name?, knowledge?, images?, max_images?)` returns a JSON object with
 `technical_description`, `experimental_conclusion`, `experimental_significance`,
 `fit_quality`, `parameter_extraction`, and `success`. The parsed object is also
 stored in the tool result's `details.diagnosis`; when the model did not answer
@@ -24,6 +24,13 @@ figures of the conversation, so the planner does not have to pass image bytes;
 `max_images` (default 1, up to 4) says how many of the newest figures to supply,
 for example 2 when an expected reference figure precedes the measured result.
 
+`knowledge` is the host's reference for the experiment: what a good result
+looks like, the questions a reviewer asks, and the common failure patterns. A
+host that keeps such knowledge per task fills it in from `task_name`, so the
+planner only names the task; the text is sent before the context. The exact
+user turn the model received (reference, context, question) is stored in
+`details.prompt`, so a host can show what the model saw.
+
 ## Configuration
 
 No setting is required. The evaluation model is resolved in this order:

package/extensions/qcaleval.ts

@@ -2,7 +2,8 @@
  * pi-qcaleval: evaluate calibration plots with a dedicated calibration VLM.
  *
  * The extension has no QDash dependency. It takes a plot (plus the experiment
- * context the planner already knows) and asks the evaluation model, which is
+ * context the planner already knows, and the host's reference knowledge for
+ * the task when the host supplies one) and asks the evaluation model, which is
  * configured separately from the conversation's model, for a diagnosis. The
  * planner decides what to do with it; whether a calibration loop continues is
  * the host's policy, not this tool's.
@@ -109,20 +110,31 @@ export const SYSTEM_PROMPT = [
   '  "fit_quality": "good" | "marginal" | "poor" and why;',
   '  "parameter_extraction": extracted values with units, or null;',
   '  "success": true | false, whether the experiment achieved its goal.',
-  "Base every statement on the plot and the given context only.",
+  "Base every statement on the plot, the reference, and the given context only.",
 ].join("\n");
 
 /**
- * Build the single user turn sent to the evaluation model. With several
- * figures, the model is told how many follow so it reads them in the order
- * the context describes (expected reference first, measured result last).
+ * Build the single user turn sent to the evaluation model. The host's
+ * reference for the task, when given, comes first so the context and the plot
+ * are read against it. With several figures, the model is told how many follow
+ * so it reads them in the order the context describes (expected reference
+ * first, measured result last).
  */
 export function buildUserPrompt(
   context: string,
   question: string | undefined,
   imageCount = 1,
+  knowledge?: string,
 ): string {
-  const lines = ["Experiment context:", context.trim() || "(none)"];
+  const lines: string[] = [];
+  if (knowledge?.trim()) {
+    lines.push(
+      "Reference for this experiment (expected result, review questions, failure patterns):",
+      knowledge.trim(),
+      "",
+    );
+  }
+  lines.push("Experiment context:", context.trim() || "(none)");
   if (imageCount > 1) {
     lines.push(
       "",
@@ -130,6 +142,9 @@ export function buildUserPrompt(
     );
   }
   if (question?.trim()) lines.push("", "Specific question:", question.trim());
+  // Repeated here because a long reference ahead of the plot pulls the model
+  // towards prose; the last instruction it reads is the output format.
+  lines.push("", "Answer with the JSON object described in the system prompt, and nothing else.");
   return lines.join("\n");
 }
 
@@ -199,6 +214,18 @@ const parameters = Type.Object({
   question: Type.Optional(
     Type.String({ description: "A specific question to answer about the plot" }),
   ),
+  task_name: Type.Optional(
+    Type.String({
+      description:
+        "The calibration task's name as the host knows it (e.g. RandomizedBenchmarking). Lets the host supply its reference knowledge for the experiment.",
+    }),
+  ),
+  knowledge: Type.Optional(
+    Type.String({
+      description:
+        "Normally omitted: the host supplies the reference for `task_name` (expected result, review questions, failure patterns). Fill only to override it with text you were given.",
+    }),
+  ),
   images: Type.Optional(
     Type.Array(imageSchema, {
       description:
@@ -224,6 +251,8 @@ export interface EvaluationDetails {
   diagnosis?: Record<string, unknown> | null;
   /** The model's answer when it was not the requested JSON. */
   raw?: string;
+  /** The user turn the model was sent (reference, context, question), so the host can show what it saw. */
+  prompt?: string;
 }
 
 function failure(spec: ModelSpec | undefined, text: string): AgentToolResult<EvaluationDetails> {
@@ -244,6 +273,7 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
     promptGuidelines: [
       "Use qcal_evaluate for every calibration figure you need to judge, instead of judging it yourself.",
       "Pass the task name, target, and expected result as context so the diagnosis is specific.",
+      "Pass task_name when you know it: the host then supplies its reference knowledge for that task.",
       "Treat the returned diagnosis as the evaluation of record; add your own reasoning only on top of it.",
     ],
     parameters,
@@ -267,6 +297,7 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
         return failure(spec, "no plot was given. Pass the figure in `images` or fetch it first.");
       }
 
+      const prompt = buildUserPrompt(params.context, params.question, images.length, params.knowledge);
       const message = await ctx.modelRegistry.complete(
         model,
         {
@@ -275,10 +306,7 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
             {
               role: "user",
               content: [
-                {
-                  type: "text",
-                  text: buildUserPrompt(params.context, params.question, images.length),
-                },
+                { type: "text", text: prompt },
                 ...images,
               ],
               timestamp: Date.now(),
@@ -299,6 +327,7 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
           ...(fallback ? { fallback: true } : {}),
           diagnosis: diagnosis ?? null,
           ...(diagnosis ? {} : { raw: text }),
+          prompt,
         },
         ...(message.usage ? { usage: message.usage } : {}),
       };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orangekame3/pi-qcaleval",
-  "version": "0.0.2-beta.3",
+  "version": "0.0.2-beta.5",
   "description": "pi extension that sends calibration plots to a calibration-evaluation VLM such as NVIDIA Ising-Calibration and returns a structured diagnosis",
   "license": "Apache-2.0",
   "repository": {

package/skills/qcaleval/SKILL.md

@@ -18,6 +18,10 @@ Ising-Calibration by default) and returns a JSON diagnosis with
   the task name, the target (qubit or coupling), the swept parameter, and what a
   successful result looks like. Add `question` when a specific point matters
   (for example "is the chevron centred on the current frequency?").
+- Pass `task_name` whenever you know the task (for example `CheckRabi`): the
+  host then supplies its own reference knowledge for that experiment, which is
+  more reliable than a summary you write yourself. Keep `context` for what the
+  reference cannot know: the target, the swept range, and previous values.
 - When the conversation holds an expected reference figure followed by the
   measured result (a task review opens this way), pass `max_images: 2` and say
   in `context` that the first figure is the reference and the last the