@orangekame3/pi-qcaleval 0.0.1 → 0.0.2-beta.3

package/README.md

@@ -26,10 +26,15 @@ for example 2 when an expected reference figure precedes the measured result.
 
 ## Configuration
 
-| Variable | Purpose |
-|----------|---------|
-| `QCALEVAL_MODEL` | `provider/modelId` of the evaluation model in pi's `models.json` (default: `vllm/nvidia/Ising-Calibration-1.5-31B-NVFP4`) |
-| `QCALEVAL_MAX_TOKENS` | Output budget for one evaluation (default: `2048`) |
+No setting is required. The evaluation model is resolved in this order:
+
+1. `QCALEVAL_MODEL` (`provider/modelId`), an explicit override for pi CLI use;
+2. a configured calibration VLM, recognised by its id (`Ising-Calibration`);
+3. the conversation's own model, when it accepts images. The result is then
+   marked `fallback: true` in `details`, so a host without an Ising deployment
+   still gets a diagnosis from the vision model it already runs.
+
+The output budget follows the model's `maxTokens` (at most 4096).
 
 The provider must be declared in pi's `models.json` with its base URL and
 credential, for example:

package/extensions/qcaleval.ts

@@ -16,20 +16,25 @@ import type {
 } from "@earendil-works/pi-coding-agent";
 import { Type, type Static } from "typebox";
 
-/** `provider/modelId`; the model id itself may contain slashes (nvidia/...). */
-export const DEFAULT_MODEL = "vllm/nvidia/Ising-Calibration-1.5-31B-NVFP4";
-
 /** Figures per evaluation; a 16k-token deployment holds a few plots comfortably. */
 export const MAX_IMAGES = 4;
 
+/** Model ids of models fine-tuned for calibration plots; preferred when configured. */
+export const CALIBRATION_MODEL_PATTERN = /ising-calibration/i;
+
+/** Output budget when the model declares none; a diagnosis is a few hundred tokens. */
+const DEFAULT_MAX_TOKENS = 2048;
+const MAX_OUTPUT_TOKENS = 4096;
+
 export interface ModelSpec {
   provider: string;
   modelId: string;
 }
 
-/** Parse `provider/modelId`, splitting at the first slash only. */
-export function parseModelSpec(value: string | undefined): ModelSpec {
-  const spec = (value ?? "").trim() || DEFAULT_MODEL;
+/** Parse an explicit `provider/modelId` override; the model id may contain slashes (nvidia/...). */
+export function parseModelSpec(value: string | undefined): ModelSpec | undefined {
+  const spec = (value ?? "").trim();
+  if (!spec) return undefined;
   const slash = spec.indexOf("/");
   if (slash <= 0 || slash === spec.length - 1) {
     throw new Error(`QCALEVAL_MODEL must be provider/modelId, got "${spec}"`);
@@ -37,6 +42,54 @@ export function parseModelSpec(value: string | undefined): ModelSpec {
   return { provider: spec.slice(0, slash), modelId: spec.slice(slash + 1) };
 }
 
+/** The slice of pi's model registry the resolution needs. */
+export interface ModelLookup {
+  find(provider: string, modelId: string): Model<Api> | undefined;
+  getAvailable(): Model<Api>[];
+}
+
+export type EvaluationModel =
+  | { model: Model<Api>; fallback: false }
+  | { model: Model<Api>; fallback: true }
+  | { model?: undefined; error: string };
+
+/**
+ * Pick the model that judges the plot, without any host-specific setting:
+ *
+ * 1. `QCALEVAL_MODEL` (`provider/modelId`), an explicit override for the pi CLI;
+ * 2. a configured calibration VLM, recognised by its id (Ising-Calibration);
+ * 3. the conversation's own model, when it accepts images.
+ *
+ * A host that declares the calibration model in its models.json gets it
+ * everywhere; one that does not (no Ising deployment) still gets a diagnosis
+ * from the vision model it already runs, marked as a fallback.
+ */
+export function resolveEvaluationModel(
+  registry: ModelLookup,
+  current: Model<Api> | undefined,
+  override: string | undefined,
+): EvaluationModel {
+  const spec = parseModelSpec(override);
+  if (spec) {
+    const model = registry.find(spec.provider, spec.modelId);
+    return model
+      ? { model, fallback: false }
+      : { error: `model ${spec.provider}/${spec.modelId} is not configured (QCALEVAL_MODEL)` };
+  }
+  const calibration = registry.getAvailable().find((model) => CALIBRATION_MODEL_PATTERN.test(model.id));
+  if (calibration) return { model: calibration, fallback: false };
+  if (current?.input.includes("image")) return { model: current, fallback: true };
+  return {
+    error:
+      "no evaluation model: declare a calibration VLM (Ising-Calibration) in models.json, or run with a vision model",
+  };
+}
+
+/** Output budget for one evaluation, within what the model allows. */
+export function outputBudget(model: Pick<Model<Api>, "maxTokens">): number {
+  return model.maxTokens > 0 ? Math.min(model.maxTokens, MAX_OUTPUT_TOKENS) : DEFAULT_MAX_TOKENS;
+}
+
 /** Categories the Ising-Calibration model card describes; asked for in this order. */
 export const DIAGNOSIS_SECTIONS = [
   "technical_description",
@@ -164,14 +217,21 @@ const parameters = Type.Object({
 
 /** Stored on the tool result; the host reads `diagnosis` without reparsing text. */
 export interface EvaluationDetails {
-  model: ModelSpec;
+  /** The model that judged the plot; absent when none could be resolved. */
+  model?: ModelSpec;
+  /** True when no calibration VLM was configured and the conversation's model judged instead. */
+  fallback?: boolean;
   diagnosis?: Record<string, unknown> | null;
   /** The model's answer when it was not the requested JSON. */
   raw?: string;
 }
 
-function failure(spec: ModelSpec, text: string): AgentToolResult<EvaluationDetails> {
-  return { content: [{ type: "text", text: `qcal_evaluate: ${text}` }], details: { model: spec }, isError: true };
+function failure(spec: ModelSpec | undefined, text: string): AgentToolResult<EvaluationDetails> {
+  return {
+    content: [{ type: "text", text: `qcal_evaluate: ${text}` }],
+    details: spec ? { model: spec } : {},
+    isError: true,
+  };
 }
 
 export default function qcalevalExtension(pi: ExtensionAPI) {
@@ -194,11 +254,10 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
       _onUpdate: unknown,
       ctx: ExtensionToolContext,
     ): Promise<AgentToolResult<EvaluationDetails>> {
-      const spec = parseModelSpec(process.env.QCALEVAL_MODEL);
-      const model: Model<Api> | undefined = ctx.modelRegistry.find(spec.provider, spec.modelId);
-      if (!model) {
-        return failure(spec, `model ${spec.provider}/${spec.modelId} is not configured (QCALEVAL_MODEL)`);
-      }
+      const resolved = resolveEvaluationModel(ctx.modelRegistry, ctx.model, process.env.QCALEVAL_MODEL);
+      if (!resolved.model) return failure(undefined, resolved.error);
+      const { model, fallback } = resolved;
+      const spec: ModelSpec = { provider: model.provider, modelId: model.id };
       const images: ImageContent[] = (params.images ?? []).map((image) => ({
         type: "image",
         data: image.data,
@@ -226,7 +285,7 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
             },
           ],
         },
-        { signal, maxTokens: Number(process.env.QCALEVAL_MAX_TOKENS ?? 2048) },
+        { signal, maxTokens: outputBudget(model) },
       );
       if (message.stopReason === "error") {
         return failure(spec, message.errorMessage ?? "model error");
@@ -237,6 +296,7 @@ export default function qcalevalExtension(pi: ExtensionAPI) {
         content: [{ type: "text", text: diagnosis ? JSON.stringify(diagnosis, null, 2) : text }],
         details: {
           model: spec,
+          ...(fallback ? { fallback: true } : {}),
           diagnosis: diagnosis ?? null,
           ...(diagnosis ? {} : { raw: text }),
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orangekame3/pi-qcaleval",
-  "version": "0.0.1",
+  "version": "0.0.2-beta.3",
   "description": "pi extension that sends calibration plots to a calibration-evaluation VLM such as NVIDIA Ising-Calibration and returns a structured diagnosis",
   "license": "Apache-2.0",
   "repository": {