@orangekame3/pi-qcaleval 0.0.2-beta.11 → 0.0.2-beta.12

package/README.md

@@ -37,7 +37,10 @@ every user turn the model received (with `[figure]` where an image was), and
 `details.calls` each question's call with its duration, stop reason and output
 tokens. A call that fails is retried once, as the benchmark retries API
 failures; nothing is re-asked for its content. Questions that got no answer are
-listed in `details.missing_keys`. `questions` asks a subset.
+listed in `details.missing_keys`. `questions` asks a subset. `dry_run` builds
+the prompts and returns them (`details.prompts` by question, `details.prompt`
+as one text) without asking the model, so a host can show what a task's
+reference and examples turn into; the figure may be omitted then.
 
 `knowledge` is the experiment background the benchmark prepends to Q2–Q6: a few
 sentences on what the experiment sweeps and what a successful result looks like.

package/extensions/qcaleval.ts

@@ -769,6 +769,12 @@ const parameters = Type.Object({
       description: "The questions to ask (default: all six, in order).",
     }),
   ),
+  dry_run: Type.Optional(
+    Type.Boolean({
+      description:
+        "Build the prompts and return them without asking the model, to show what it would be sent. The figure may be omitted.",
+    }),
+  ),
 });
 
 /** Stored on the tool result; the host reads `diagnosis` without reparsing text. */
@@ -793,8 +799,12 @@ export interface EvaluationDetails {
   calls?: CallRecord[];
   /** The user turns the model was sent, one per question, so the host can show what it saw. */
   prompt?: string;
+  /** The same turns by question. */
+  prompts?: Partial<Record<Question, string>>;
   /** How many labeled examples were shown. */
   examples_used?: number;
+  /** True when the prompts were built but the model was not asked. */
+  dry_run?: boolean;
 }
 
 function failure(spec: ModelSpec | undefined, text: string): AgentToolResult<EvaluationDetails> {
@@ -828,16 +838,20 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
       ctx: ExtensionToolContext,
     ): Promise<AgentToolResult<EvaluationDetails>> {
       const resolved = resolveEvaluationModel(ctx.modelRegistry, ctx.model, process.env.QCALEVAL_MODEL);
-      if (!resolved.model) return failure(undefined, resolved.error);
-      const { model, fallback } = resolved;
-      const spec: ModelSpec = { provider: model.provider, modelId: model.id };
+      const dryRun = params.dry_run === true;
+      if (!resolved.model && !dryRun) return failure(undefined, resolved.error);
+      const spec: ModelSpec | undefined = resolved.model
+        ? { provider: resolved.model.provider, modelId: resolved.model.id }
+        : undefined;
       const images: ImageContent[] = (params.images ?? []).map((image) => ({
         type: "image",
         data: image.data,
         mimeType: image.mimeType,
       }));
       if (images.length === 0) {
-        return failure(spec, "no plot was given. Pass the figure in `images` or fetch it first.");
+        if (!dryRun) return failure(spec, "no plot was given. Pass the figure in `images` or fetch it first.");
+        // The prompts are the same whatever the figure; a placeholder stands in.
+        images.push({ type: "image", data: "", mimeType: "image/png" });
       }
 
       const requested = new Set<string>(params.questions ?? []);
@@ -852,6 +866,25 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
       };
       const turns = new Map(questions.map((question) => [question, buildQuestionContent(question, input)]));
       const examplesUsed = fitExamples(input.examples ?? []).length;
+      const prompts = Object.fromEntries(questions.map((question) => [question, turns.get(question)!.prompt])) as Partial<
+        Record<Question, string>
+      >;
+      const prompt = promptRecord(questions.map((question) => ({ question, prompt: turns.get(question)!.prompt })));
+      if (dryRun || !resolved.model) {
+        return {
+          content: [{ type: "text", text: prompt }],
+          details: {
+            ...(spec ? { model: spec } : {}),
+            ...(resolved.model && resolved.fallback ? { fallback: true } : {}),
+            dry_run: true,
+            questions,
+            prompt,
+            prompts,
+            ...(examplesUsed ? { examples_used: examplesUsed } : {}),
+          },
+        };
+      }
+      const { model, fallback } = resolved;
       const answers = await askQuestions(
         questions,
         (question) =>
@@ -889,7 +922,8 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
           attempts: answers.attempts,
           stop_reason: answers.stopReason,
           calls: answers.calls,
-          prompt: promptRecord(questions.map((question) => ({ question, prompt: turns.get(question)!.prompt }))),
+          prompt,
+          prompts,
           ...(examplesUsed ? { examples_used: examplesUsed } : {}),
         },
         ...(answers.usage ? { usage: answers.usage } : {}),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orangekame3/pi-qcaleval",
-  "version": "0.0.2-beta.11",
+  "version": "0.0.2-beta.12",
   "description": "pi extension that sends calibration plots to a calibration-evaluation VLM such as NVIDIA Ising-Calibration and returns a structured diagnosis",
   "license": "Apache-2.0",
   "repository": {

package/skills/qcaleval/SKILL.md

@@ -33,6 +33,9 @@ and `calibration_diagnosis` (a status code, a suggested range, notes), plus
   measurement. The host then supplies both; the default is the newest figure only.
 - Pass `questions` to ask a subset, for example `["calibration_diagnosis"]`
   when only the status matters.
+- Pass `dry_run: true` to see the prompts the model would get (per question)
+  without asking it, for example when the user wants to check a task's
+  reference before judging.
 
 ## Labeled examples (in-context)