@plune-ai/cli 0.32.1 → 0.33.0

package/dist/index.cjs

@@ -457,24 +457,24 @@ function priceFrom(price, usage) {
 var defaultWarn = (message) => {
   process.stderr.write(message + "\n");
 };
-function resolveCost(usage, model, reportedCostUsd, overrides, deps = {}) {
+function resolveCostEntry(usage, model, reportedCostUsd, overrides, deps = {}) {
   const table = deps.table ?? PRICE_TABLE;
   const warn = deps.warn ?? defaultWarn;
   const override = overrides?.[model];
   if (override !== void 0) {
-    return priceFrom(override, usage);
+    return { cost_usd: priceFrom(override, usage), priced: true };
   }
   if (reportedCostUsd !== void 0) {
-    return reportedCostUsd;
+    return { cost_usd: reportedCostUsd, priced: true };
   }
   const tablePrice = table[model];
   if (tablePrice !== void 0) {
-    return priceFrom(tablePrice, usage);
+    return { cost_usd: priceFrom(tablePrice, usage), priced: true };
   }
   warn(
     `plune: no price for model "${model}" \u2014 reporting cost_usd=0. Set pricing["${model}"] in plune.yaml to track its cost.`
   );
-  return 0;
+  return { cost_usd: 0, priced: false };
 }
 
 // src/providers/anthropic.ts
@@ -520,7 +520,7 @@ function makeAnthropicProvider(config, env, pricing) {
       }
     },
     estimateCost(usage, reportedCostUsd) {
-      return { cost_usd: resolveCost(usage, config.model, reportedCostUsd, pricing) };
+      return resolveCostEntry(usage, config.model, reportedCostUsd, pricing);
     }
   };
 }
@@ -576,7 +576,7 @@ function makeOpenAiCompatibleProvider(opts) {
       }
     },
     estimateCost(usage, reportedCostUsd) {
-      return { cost_usd: resolveCost(usage, config.model, reportedCostUsd, pricing) };
+      return resolveCostEntry(usage, config.model, reportedCostUsd, pricing);
     }
   };
 }
@@ -638,9 +638,7 @@ function makeEstimateProvider(config, pricing) {
     complete: () => Promise.reject(
       new Error("The --dry-run provider only estimates cost; it never calls a model.")
     ),
-    estimateCost: (usage, reportedCostUsd) => ({
-      cost_usd: resolveCost(usage, config.model, reportedCostUsd, pricing)
-    })
+    estimateCost: (usage, reportedCostUsd) => resolveCostEntry(usage, config.model, reportedCostUsd, pricing)
   };
 }
 
@@ -1426,6 +1424,12 @@ function toRunError(err) {
 function estimateUsage(prompt5, maxTokens) {
   return { input_tokens: Math.ceil(prompt5.length / 4), output_tokens: maxTokens };
 }
+function pricedField(priced) {
+  return priced === void 0 ? {} : { priced };
+}
+function costFields(c) {
+  return { cost_usd: c.cost_usd, ...pricedField(c.priced) };
+}
 async function runRow(p) {
   const t0 = p.now();
   const prompt5 = resolvePrompt(p.template, p.row.vars);
@@ -1433,7 +1437,8 @@ async function runRow(p) {
   const maxTokens = p.providerConfig.max_tokens ?? DEFAULT_MAX_TOKENS2;
   if (p.dryRun) {
     const est = estimateUsage(prompt5, maxTokens);
-    const usage2 = { ...est, cost_usd: p.provider.estimateCost(est).cost_usd };
+    const cost = p.provider.estimateCost(est);
+    const usage2 = { ...est, ...costFields(cost) };
     return { vars: p.row.vars, output: null, cached: false, usage: usage2, assertions: [] };
   }
   const key = cacheKey(
@@ -1471,7 +1476,9 @@ async function runRow(p) {
     usage = {
       input_tokens: hit.usage.input_tokens,
       output_tokens: hit.usage.output_tokens,
-      cost_usd: 0
+      cost_usd: 0,
+      // A hit costs nothing, but whether that 0 is a known price depends on the model.
+      ...pricedField(p.provider.estimateCost(hit.usage).priced)
     };
     cacheSaved = {
       input_tokens: hit.usage.input_tokens,
@@ -1488,12 +1495,13 @@ async function runRow(p) {
         prompt_resolved: prompt5
       });
       output = res.output;
+      const cost = p.provider.estimateCost(res.usage, res.cost_usd);
       usage = {
         input_tokens: res.usage.input_tokens,
         output_tokens: res.usage.output_tokens,
         // Prefer the provider-reported actual cost (res.cost_usd) when present; resolveCost still
         // lets a config pricing override win, else falls back to the table estimate (ADR-PRC01).
-        cost_usd: p.provider.estimateCost(res.usage, res.cost_usd).cost_usd
+        ...costFields(cost)
       };
       if (!p.noCache && isCacheable(res) && answerParses(res.output, p.assertions)) {
         p.cache.set(key, res);
@@ -1510,8 +1518,11 @@ async function runRow(p) {
     }
   }
   let judgeCost = 0;
+  let judgeUnpriced = false;
   const judge = buildJudge(p.provider, p.providerConfig, (u) => {
-    judgeCost += p.provider.estimateCost(u).cost_usd;
+    const c = p.provider.estimateCost(u);
+    judgeCost += c.cost_usd;
+    if (c.priced === false) judgeUnpriced = true;
   });
   const records = [];
   let assertionError;
@@ -1536,7 +1547,12 @@ async function runRow(p) {
       break;
     }
   }
-  const finalUsage = { ...usage, cost_usd: usage.cost_usd + judgeCost };
+  const finalUsage = {
+    ...usage,
+    cost_usd: usage.cost_usd + judgeCost,
+    // A row is priced only when every call in it was; the field stays absent if the provider never said.
+    ...judgeUnpriced ? { priced: false } : {}
+  };
   return {
     vars: p.row.vars,
     output,
@@ -1758,7 +1774,9 @@ var assertionResultRecordSchema2 = import_zod3.z.object({
 var usageSchema = import_zod3.z.object({
   input_tokens: import_zod3.z.number(),
   output_tokens: import_zod3.z.number(),
-  cost_usd: import_zod3.z.number()
+  cost_usd: import_zod3.z.number(),
+  /** cli#151. Optional and additive — a RunResult written before it validates unchanged. */
+  priced: import_zod3.z.boolean().optional()
 });
 var runErrorSchema = import_zod3.z.object({
   code: import_zod3.z.string(),

package/dist/index.d.cts

@@ -44,6 +44,12 @@ interface Usage {
     input_tokens: number;
     output_tokens: number;
     cost_usd: number;
+    /**
+     * plune-ai/cli#151. `true` when the price of the model is known, a zero price included (a `:free`
+     * model, `pricing: 0`); `false` when nobody has a price and `cost_usd` is a placeholder 0. Optional
+     * and additive: a report without it reads as before, and `schemaVersion` stays 1.
+     */
+    priced?: boolean;
 }
 interface RunError {
     code: string;
@@ -263,6 +269,8 @@ interface CompletionResponse {
 }
 interface CostEstimate {
     cost_usd: number;
+    /** Whether `cost_usd` is a known price (zero included) rather than the 0 of an unpriced model. Optional and additive. */
+    priced?: boolean;
 }
 interface Provider<_TConfig extends ProviderConfig = ProviderConfig> {
     complete(req: CompletionRequest): Promise<CompletionResponse>;
@@ -510,6 +518,7 @@ declare const runResultSchema: z.ZodObject<{
                 input_tokens: z.ZodNumber;
                 output_tokens: z.ZodNumber;
                 cost_usd: z.ZodNumber;
+                priced: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strip>>;
             latency_ms: z.ZodOptional<z.ZodNumber>;
             error: z.ZodOptional<z.ZodObject<{

package/dist/index.d.ts

@@ -44,6 +44,12 @@ interface Usage {
     input_tokens: number;
     output_tokens: number;
     cost_usd: number;
+    /**
+     * plune-ai/cli#151. `true` when the price of the model is known, a zero price included (a `:free`
+     * model, `pricing: 0`); `false` when nobody has a price and `cost_usd` is a placeholder 0. Optional
+     * and additive: a report without it reads as before, and `schemaVersion` stays 1.
+     */
+    priced?: boolean;
 }
 interface RunError {
     code: string;
@@ -263,6 +269,8 @@ interface CompletionResponse {
 }
 interface CostEstimate {
     cost_usd: number;
+    /** Whether `cost_usd` is a known price (zero included) rather than the 0 of an unpriced model. Optional and additive. */
+    priced?: boolean;
 }
 interface Provider<_TConfig extends ProviderConfig = ProviderConfig> {
     complete(req: CompletionRequest): Promise<CompletionResponse>;
@@ -510,6 +518,7 @@ declare const runResultSchema: z.ZodObject<{
                 input_tokens: z.ZodNumber;
                 output_tokens: z.ZodNumber;
                 cost_usd: z.ZodNumber;
+                priced: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strip>>;
             latency_ms: z.ZodOptional<z.ZodNumber>;
             error: z.ZodOptional<z.ZodObject<{

package/dist/index.js

@@ -397,24 +397,24 @@ function priceFrom(price, usage) {
 var defaultWarn = (message) => {
   process.stderr.write(message + "\n");
 };
-function resolveCost(usage, model, reportedCostUsd, overrides, deps = {}) {
+function resolveCostEntry(usage, model, reportedCostUsd, overrides, deps = {}) {
   const table = deps.table ?? PRICE_TABLE;
   const warn = deps.warn ?? defaultWarn;
   const override = overrides?.[model];
   if (override !== void 0) {
-    return priceFrom(override, usage);
+    return { cost_usd: priceFrom(override, usage), priced: true };
   }
   if (reportedCostUsd !== void 0) {
-    return reportedCostUsd;
+    return { cost_usd: reportedCostUsd, priced: true };
   }
   const tablePrice = table[model];
   if (tablePrice !== void 0) {
-    return priceFrom(tablePrice, usage);
+    return { cost_usd: priceFrom(tablePrice, usage), priced: true };
   }
   warn(
     `plune: no price for model "${model}" \u2014 reporting cost_usd=0. Set pricing["${model}"] in plune.yaml to track its cost.`
   );
-  return 0;
+  return { cost_usd: 0, priced: false };
 }
 
 // src/providers/anthropic.ts
@@ -460,7 +460,7 @@ function makeAnthropicProvider(config, env, pricing) {
       }
     },
     estimateCost(usage, reportedCostUsd) {
-      return { cost_usd: resolveCost(usage, config.model, reportedCostUsd, pricing) };
+      return resolveCostEntry(usage, config.model, reportedCostUsd, pricing);
     }
   };
 }
@@ -516,7 +516,7 @@ function makeOpenAiCompatibleProvider(opts) {
       }
     },
     estimateCost(usage, reportedCostUsd) {
-      return { cost_usd: resolveCost(usage, config.model, reportedCostUsd, pricing) };
+      return resolveCostEntry(usage, config.model, reportedCostUsd, pricing);
     }
   };
 }
@@ -578,9 +578,7 @@ function makeEstimateProvider(config, pricing) {
     complete: () => Promise.reject(
       new Error("The --dry-run provider only estimates cost; it never calls a model.")
     ),
-    estimateCost: (usage, reportedCostUsd) => ({
-      cost_usd: resolveCost(usage, config.model, reportedCostUsd, pricing)
-    })
+    estimateCost: (usage, reportedCostUsd) => resolveCostEntry(usage, config.model, reportedCostUsd, pricing)
   };
 }
 
@@ -1366,6 +1364,12 @@ function toRunError(err) {
 function estimateUsage(prompt5, maxTokens) {
   return { input_tokens: Math.ceil(prompt5.length / 4), output_tokens: maxTokens };
 }
+function pricedField(priced) {
+  return priced === void 0 ? {} : { priced };
+}
+function costFields(c) {
+  return { cost_usd: c.cost_usd, ...pricedField(c.priced) };
+}
 async function runRow(p) {
   const t0 = p.now();
   const prompt5 = resolvePrompt(p.template, p.row.vars);
@@ -1373,7 +1377,8 @@ async function runRow(p) {
   const maxTokens = p.providerConfig.max_tokens ?? DEFAULT_MAX_TOKENS2;
   if (p.dryRun) {
     const est = estimateUsage(prompt5, maxTokens);
-    const usage2 = { ...est, cost_usd: p.provider.estimateCost(est).cost_usd };
+    const cost = p.provider.estimateCost(est);
+    const usage2 = { ...est, ...costFields(cost) };
     return { vars: p.row.vars, output: null, cached: false, usage: usage2, assertions: [] };
   }
   const key = cacheKey(
@@ -1411,7 +1416,9 @@ async function runRow(p) {
     usage = {
       input_tokens: hit.usage.input_tokens,
       output_tokens: hit.usage.output_tokens,
-      cost_usd: 0
+      cost_usd: 0,
+      // A hit costs nothing, but whether that 0 is a known price depends on the model.
+      ...pricedField(p.provider.estimateCost(hit.usage).priced)
     };
     cacheSaved = {
       input_tokens: hit.usage.input_tokens,
@@ -1428,12 +1435,13 @@ async function runRow(p) {
         prompt_resolved: prompt5
       });
       output = res.output;
+      const cost = p.provider.estimateCost(res.usage, res.cost_usd);
       usage = {
         input_tokens: res.usage.input_tokens,
         output_tokens: res.usage.output_tokens,
         // Prefer the provider-reported actual cost (res.cost_usd) when present; resolveCost still
         // lets a config pricing override win, else falls back to the table estimate (ADR-PRC01).
-        cost_usd: p.provider.estimateCost(res.usage, res.cost_usd).cost_usd
+        ...costFields(cost)
       };
       if (!p.noCache && isCacheable(res) && answerParses(res.output, p.assertions)) {
         p.cache.set(key, res);
@@ -1450,8 +1458,11 @@ async function runRow(p) {
     }
   }
   let judgeCost = 0;
+  let judgeUnpriced = false;
   const judge = buildJudge(p.provider, p.providerConfig, (u) => {
-    judgeCost += p.provider.estimateCost(u).cost_usd;
+    const c = p.provider.estimateCost(u);
+    judgeCost += c.cost_usd;
+    if (c.priced === false) judgeUnpriced = true;
   });
   const records = [];
   let assertionError;
@@ -1476,7 +1487,12 @@ async function runRow(p) {
       break;
     }
   }
-  const finalUsage = { ...usage, cost_usd: usage.cost_usd + judgeCost };
+  const finalUsage = {
+    ...usage,
+    cost_usd: usage.cost_usd + judgeCost,
+    // A row is priced only when every call in it was; the field stays absent if the provider never said.
+    ...judgeUnpriced ? { priced: false } : {}
+  };
   return {
     vars: p.row.vars,
     output,
@@ -1698,7 +1714,9 @@ var assertionResultRecordSchema2 = z3.object({
 var usageSchema = z3.object({
   input_tokens: z3.number(),
   output_tokens: z3.number(),
-  cost_usd: z3.number()
+  cost_usd: z3.number(),
+  /** cli#151. Optional and additive — a RunResult written before it validates unchanged. */
+  priced: z3.boolean().optional()
 });
 var runErrorSchema = z3.object({
   code: z3.string(),

package/dist/reporter-core.cjs

@@ -726,6 +726,7 @@ function boundKey(key) {
   if (/[\uD800-\uDBFF]$/.test(head)) head = head.slice(0, -1);
   return { ...key, value: `${head}#${digest}` };
 }
+var keyId = (key) => `${key.kind ?? ""}\0${key.value}`;
 function whyUnoffered(out) {
   if (out.ok || out.status !== 429) return "other";
   if (/review queue quota reached/i.test(out.detail)) return "queue";
@@ -863,7 +864,7 @@ async function startRun(passed, declared) {
     stats.deferred += results.length;
   }
   async function resolveKeys(values) {
-    const unknown = values.filter((k) => !resolved.has(k.value));
+    const unknown = values.filter((k) => !resolved.has(keyId(k)));
     if (unknown.length === 0 || offline) return;
     for (const part of chunk(unknown, RESOLVE_CHUNK)) {
       const out = await client.post(
@@ -875,7 +876,7 @@ async function startRun(passed, declared) {
         return;
       }
       for (const row of out.body.results) {
-        if (typeof row?.key?.value === "string") resolved.set(row.key.value, row.testCaseId);
+        if (typeof row?.key?.value === "string") resolved.set(keyId(row.key), row.testCaseId);
       }
     }
   }
@@ -884,7 +885,7 @@ async function startRun(passed, declared) {
     const stated2 = expected?.slice(0, EXPECTED_MAX) ?? [];
     const configuration2 = stated2.length === 0 ? void 0 : {
       expected: stated2.flatMap((keys) => {
-        const hit = keys.map((k) => resolved.get(k.value)).find((v) => typeof v === "string");
+        const hit = keys.map((k) => resolved.get(keyId(k))).find((v) => typeof v === "string");
         if (typeof hit === "string") return [{ testCaseId: hit }];
         return keys[0] === void 0 ? [] : [{ externalKey: keys[0] }];
       }),
@@ -933,7 +934,7 @@ async function startRun(passed, declared) {
     if (result.testCaseId !== void 0) return { kind: "found", testCaseId: result.testCaseId };
     let unanswered = false;
     for (const key of result.keys) {
-      const hit = resolved.get(key.value);
+      const hit = resolved.get(keyId(key));
       if (typeof hit === "string") return { kind: "found", testCaseId: hit };
       if (hit === void 0) unanswered = true;
     }
@@ -945,14 +946,14 @@ async function startRun(passed, declared) {
     if (result.keys.length === 0) return;
     const title = boundTitle(result.title);
     if (title !== result.title) shortened.add(result.title);
-    discoveries.set(result.keys[0].value, {
+    discoveries.set(keyId(result.keys[0]), {
       keys: result.keys,
       title,
       source: result.source,
       specRef: result.specRef,
       rawStatus: result.rawStatus
     });
-    held.set(result.keys[0].value, result);
+    for (const key of result.keys) held.set(keyId(key), result);
   }
   async function offer() {
     if (discoveries.size === 0 || offline || runId === null) return;
@@ -976,9 +977,9 @@ async function startRun(passed, declared) {
       const submissions = [];
       const filesByKey = /* @__PURE__ */ new Map();
       for (const r of made) {
-        const pending = typeof r.id === "string" && r.id !== "" ? held.get(r.key?.value) : void 0;
+        const pending = typeof r.id === "string" && r.id !== "" && typeof r.key?.value === "string" ? held.get(keyId(r.key)) : void 0;
         if (pending === void 0) continue;
-        held.delete(r.key.value);
+        for (const key of pending.keys) held.delete(keyId(key));
         const { keys: _keys, files, ...rest } = pending;
         submissions.push({ ...rest, testCaseId: r.id });
         if (files !== void 0) filesByKey.set(pending.resultKey, files);

package/dist/reporter-core.js

@@ -676,6 +676,7 @@ function boundKey(key) {
   if (/[\uD800-\uDBFF]$/.test(head)) head = head.slice(0, -1);
   return { ...key, value: `${head}#${digest}` };
 }
+var keyId = (key) => `${key.kind ?? ""}\0${key.value}`;
 function whyUnoffered(out) {
   if (out.ok || out.status !== 429) return "other";
   if (/review queue quota reached/i.test(out.detail)) return "queue";
@@ -813,7 +814,7 @@ async function startRun(passed, declared) {
     stats.deferred += results.length;
   }
   async function resolveKeys(values) {
-    const unknown = values.filter((k) => !resolved.has(k.value));
+    const unknown = values.filter((k) => !resolved.has(keyId(k)));
     if (unknown.length === 0 || offline) return;
     for (const part of chunk(unknown, RESOLVE_CHUNK)) {
       const out = await client.post(
@@ -825,7 +826,7 @@ async function startRun(passed, declared) {
         return;
       }
       for (const row of out.body.results) {
-        if (typeof row?.key?.value === "string") resolved.set(row.key.value, row.testCaseId);
+        if (typeof row?.key?.value === "string") resolved.set(keyId(row.key), row.testCaseId);
       }
     }
   }
@@ -834,7 +835,7 @@ async function startRun(passed, declared) {
     const stated2 = expected?.slice(0, EXPECTED_MAX) ?? [];
     const configuration2 = stated2.length === 0 ? void 0 : {
       expected: stated2.flatMap((keys) => {
-        const hit = keys.map((k) => resolved.get(k.value)).find((v) => typeof v === "string");
+        const hit = keys.map((k) => resolved.get(keyId(k))).find((v) => typeof v === "string");
         if (typeof hit === "string") return [{ testCaseId: hit }];
         return keys[0] === void 0 ? [] : [{ externalKey: keys[0] }];
       }),
@@ -883,7 +884,7 @@ async function startRun(passed, declared) {
     if (result.testCaseId !== void 0) return { kind: "found", testCaseId: result.testCaseId };
     let unanswered = false;
     for (const key of result.keys) {
-      const hit = resolved.get(key.value);
+      const hit = resolved.get(keyId(key));
       if (typeof hit === "string") return { kind: "found", testCaseId: hit };
       if (hit === void 0) unanswered = true;
     }
@@ -895,14 +896,14 @@ async function startRun(passed, declared) {
     if (result.keys.length === 0) return;
     const title = boundTitle(result.title);
     if (title !== result.title) shortened.add(result.title);
-    discoveries.set(result.keys[0].value, {
+    discoveries.set(keyId(result.keys[0]), {
       keys: result.keys,
       title,
       source: result.source,
       specRef: result.specRef,
       rawStatus: result.rawStatus
     });
-    held.set(result.keys[0].value, result);
+    for (const key of result.keys) held.set(keyId(key), result);
   }
   async function offer() {
     if (discoveries.size === 0 || offline || runId === null) return;
@@ -926,9 +927,9 @@ async function startRun(passed, declared) {
       const submissions = [];
       const filesByKey = /* @__PURE__ */ new Map();
       for (const r of made) {
-        const pending = typeof r.id === "string" && r.id !== "" ? held.get(r.key?.value) : void 0;
+        const pending = typeof r.id === "string" && r.id !== "" && typeof r.key?.value === "string" ? held.get(keyId(r.key)) : void 0;
         if (pending === void 0) continue;
-        held.delete(r.key.value);
+        for (const key of pending.keys) held.delete(keyId(key));
         const { keys: _keys, files, ...rest } = pending;
         submissions.push({ ...rest, testCaseId: r.id });
         if (files !== void 0) filesByKey.set(pending.resultKey, files);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plune-ai/cli",
-  "version": "0.32.1",
+  "version": "0.33.0",
   "description": "AI-powered assertion testing for LLM apps — write assertions, run them against any provider, get a pass/fail report (local, CI, or diff).",
   "keywords": [
     "ai",