@plune-ai/cli 0.27.0 → 0.28.0

package/dist/cli.cjs

@@ -622,7 +622,8 @@ function makeAnthropicProvider(config, env, pricing) {
           usage: {
             input_tokens: res.usage.input_tokens,
             output_tokens: res.usage.output_tokens
-          }
+          },
+          ...res.stop_reason === "max_tokens" ? { finish_reason: "length" } : res.stop_reason !== null && res.stop_reason !== void 0 ? { finish_reason: res.stop_reason } : {}
         };
       } catch (err) {
         throw normalizeProviderError(err, apiKey, ENV_VAR);
@@ -680,8 +681,11 @@ function makeOpenAiCompatibleProvider(opts) {
           { max_retries: maxRetries }
         );
         const reportedCost = reportsCost ? res.usage?.cost : void 0;
+        const choice = res.choices[0];
+        const finish = choice?.error !== void 0 ? "error" : choice?.finish_reason;
         return {
-          output: res.choices[0]?.message?.content ?? "",
+          output: choice?.message?.content ?? "",
+          ...typeof finish === "string" ? { finish_reason: finish } : {},
           usage: {
             input_tokens: res.usage?.prompt_tokens ?? 0,
             output_tokens: res.usage?.completion_tokens ?? 0
@@ -891,18 +895,115 @@ var init_embeddings = __esm({
   }
 });
 
+// src/util/canonical-json.ts
+function normalizeValue(value2) {
+  if (value2 === null) return null;
+  if (typeof value2 === "boolean") return value2;
+  if (typeof value2 === "number") {
+    if (!isFinite(value2)) {
+      throw new Error(
+        `canonicalJson: non-finite number rejected: ${String(value2)}`
+      );
+    }
+    return value2 === 0 ? 0 : value2;
+  }
+  if (typeof value2 === "string") {
+    return value2.normalize("NFC");
+  }
+  if (Array.isArray(value2)) {
+    return value2.map(normalizeValue);
+  }
+  if (typeof value2 === "object") {
+    const proto = Object.getPrototypeOf(value2);
+    if (proto !== null && proto !== Object.prototype) {
+      const name = value2.constructor?.name ?? "unknown";
+      throw new Error(
+        `canonicalJson: unsupported non-plain object (${name}) \u2014 only plain objects, arrays, and primitives are serializable`
+      );
+    }
+    const obj = value2;
+    const sorted = {};
+    for (const key of Object.keys(obj).sort()) {
+      sorted[key] = normalizeValue(obj[key]);
+    }
+    return sorted;
+  }
+  throw new Error(
+    `canonicalJson: unsupported value type: ${typeof value2}`
+  );
+}
+function canonicalJson(value2) {
+  return JSON.stringify(normalizeValue(value2));
+}
+var init_canonical_json = __esm({
+  "src/util/canonical-json.ts"() {
+    "use strict";
+    init_cjs_shims();
+  }
+});
+
+// src/util/hash.ts
+function sha256(input) {
+  return (0, import_node_crypto.createHash)("sha256").update(input, "utf8").digest("hex");
+}
+function cacheKey(inputs, policyVersion = CACHE_POLICY_VERSION) {
+  const { provider, model, temperature, max_tokens, prompt_resolved } = inputs;
+  return sha256(
+    canonicalJson({ provider, model, temperature, max_tokens, prompt_resolved, policy: policyVersion })
+  );
+}
+function configHash(config) {
+  return sha256(canonicalJson(config));
+}
+var import_node_crypto, CACHE_POLICY_VERSION;
+var init_hash = __esm({
+  "src/util/hash.ts"() {
+    "use strict";
+    init_cjs_shims();
+    import_node_crypto = require("crypto");
+    init_canonical_json();
+    CACHE_POLICY_VERSION = 1;
+  }
+});
+
 // src/cache/cache.ts
+function isCacheable(res) {
+  if (res.output.trim() === "") return false;
+  return res.finish_reason === void 0 || !UNCACHEABLE_FINISH.has(res.finish_reason);
+}
+function loadDatabase() {
+  try {
+    return (0, import_node_module.createRequire)(importMetaUrl)("better-sqlite3");
+  } catch {
+    if (!missingNoted) {
+      missingNoted = true;
+      console.error(
+        "plune: completion cache disabled (better-sqlite3 is not installed or failed to load). Reinstall with a C++ toolchain and Python available, or ignore: runs work without the cache."
+      );
+    }
+    return void 0;
+  }
+}
 function openCache(file, opts = {}) {
   const now = opts.now ?? Date.now;
+  const Database = loadDatabase();
+  if (Database === void 0) return new NoOpCache();
   try {
-    const db = new import_better_sqlite3.default(file);
+    const db = new Database(file);
     try {
       ensureSchema(db);
     } catch (err) {
       db.close();
       throw err;
     }
-    return new SqliteCache(db, now);
+    let stale;
+    try {
+      stale = markPolicy(db, opts.policyVersion ?? CACHE_POLICY_VERSION);
+    } catch (err) {
+      db.close();
+      throw err;
+    }
+    return new SqliteCache(db, now, stale);
   } catch {
     return new NoOpCache();
   }
@@ -917,26 +1018,45 @@ function ensureSchema(db) {
     db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)").run("schema_version", current);
   }
 }
-var import_better_sqlite3, SCHEMA_VERSION, META_DDL, COMPLETIONS_DDL, SqliteCache, NoOpCache;
+function markPolicy(db, version) {
+  const current = String(version);
+  const stored = db.prepare("SELECT value FROM meta WHERE key = 'policy_version'").get()?.value;
+  if (stored === current) return 0;
+  return db.transaction(() => {
+    const { n } = db.prepare("SELECT COUNT(*) AS n FROM completions").get();
+    db.exec("DELETE FROM completions");
+    db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)").run("policy_version", current);
+    return n;
+  })();
+}
+var import_node_module, SCHEMA_VERSION, META_DDL, COMPLETIONS_DDL, UNCACHEABLE_FINISH, missingNoted, SqliteCache, NoOpCache;
 var init_cache = __esm({
   "src/cache/cache.ts"() {
     "use strict";
     init_cjs_shims();
-    import_better_sqlite3 = __toESM(require("better-sqlite3"), 1);
+    import_node_module = require("module");
+    init_hash();
     SCHEMA_VERSION = 1;
     META_DDL = "CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);";
     COMPLETIONS_DDL = "CREATE TABLE IF NOT EXISTS completions (cache_key TEXT PRIMARY KEY, output TEXT NOT NULL, input_tokens INTEGER NOT NULL, output_tokens INTEGER NOT NULL, created_at INTEGER NOT NULL);";
+    UNCACHEABLE_FINISH = /* @__PURE__ */ new Set(["length", "max_tokens", "error", "content_filter", "pause_turn"]);
+    missingNoted = false;
     SqliteCache = class {
-      constructor(db, now) {
+      constructor(db, now, staleEntries) {
         this.db = db;
         this.now = now;
+        this.staleEntries = staleEntries;
       }
       db;
       now;
+      staleEntries;
       get(key, opts = {}) {
         try {
           const row = this.db.prepare("SELECT output, input_tokens, output_tokens, created_at FROM completions WHERE cache_key = ?").get(key);
           if (row === void 0) return void 0;
+          if (typeof row.output !== "string" || !Number.isFinite(row.input_tokens) || !Number.isFinite(row.output_tokens) || !Number.isFinite(row.created_at)) {
+            return void 0;
+          }
           if (opts.maxAgeMs !== void 0 && this.now() - row.created_at > opts.maxAgeMs) {
             return void 0;
           }
@@ -962,6 +1082,12 @@ var init_cache = __esm({
         } catch {
         }
       }
+      delete(key) {
+        try {
+          this.db.prepare("DELETE FROM completions WHERE cache_key = ?").run(key);
+        } catch {
+        }
+      }
       clear() {
         try {
           this.db.exec("DELETE FROM completions");
@@ -978,6 +1104,8 @@ var init_cache = __esm({
       }
       set() {
       }
+      delete() {
+      }
       clear() {
       }
       close() {
@@ -995,71 +1123,77 @@ var init_cache2 = __esm({
   }
 });
 
-// src/util/canonical-json.ts
-function normalizeValue(value2) {
-  if (value2 === null) return null;
-  if (typeof value2 === "boolean") return value2;
-  if (typeof value2 === "number") {
-    if (!isFinite(value2)) {
-      throw new Error(
-        `canonicalJson: non-finite number rejected: ${String(value2)}`
-      );
-    }
-    return value2 === 0 ? 0 : value2;
+// src/assertions/json-extract.ts
+function extractJson(output, mode) {
+  if (mode === "strict") {
+    return tryParse(output.trim());
   }
-  if (typeof value2 === "string") {
-    return value2.normalize("NFC");
+  const fenced = extractFencedBlock(output);
+  if (fenced !== null) {
+    const r = tryParse(fenced);
+    if (r.ok) return r;
   }
-  if (Array.isArray(value2)) {
-    return value2.map(normalizeValue);
+  for (const run of balancedRuns(output)) {
+    const r = tryParse(run);
+    if (r.ok) return r;
   }
-  if (typeof value2 === "object") {
-    const proto = Object.getPrototypeOf(value2);
-    if (proto !== null && proto !== Object.prototype) {
-      const name = value2.constructor?.name ?? "unknown";
-      throw new Error(
-        `canonicalJson: unsupported non-plain object (${name}) \u2014 only plain objects, arrays, and primitives are serializable`
-      );
-    }
-    const obj = value2;
-    const sorted = {};
-    for (const key of Object.keys(obj).sort()) {
-      sorted[key] = normalizeValue(obj[key]);
-    }
-    return sorted;
+  return { ok: false };
+}
+function tryParse(s) {
+  try {
+    return { ok: true, value: JSON.parse(s) };
+  } catch {
+    return { ok: false };
   }
-  throw new Error(
-    `canonicalJson: unsupported value type: ${typeof value2}`
-  );
 }
-function canonicalJson(value2) {
-  return JSON.stringify(normalizeValue(value2));
+function extractFencedBlock(s) {
+  const m = s.match(/```(?:json)?\s*\n?([\s\S]*?)```/i);
+  return m?.[1]?.trim() ?? null;
 }
-var init_canonical_json = __esm({
-  "src/util/canonical-json.ts"() {
-    "use strict";
-    init_cjs_shims();
+function* balancedRuns(s) {
+  let i = 0;
+  while (i < s.length) {
+    const rel = s.slice(i).search(/[{[]/);
+    if (rel === -1) return;
+    const start = i + rel;
+    const run = balancedFrom(s, start);
+    if (run === null) {
+      i = start + 1;
+    } else {
+      yield run;
+      i = start + run.length;
+    }
   }
-});
-
-// src/util/hash.ts
-function sha256(input) {
-  return (0, import_node_crypto.createHash)("sha256").update(input, "utf8").digest("hex");
-}
-function cacheKey(inputs) {
-  const { provider, model, temperature, max_tokens, prompt_resolved } = inputs;
-  return sha256(canonicalJson({ provider, model, temperature, max_tokens, prompt_resolved }));
 }
-function configHash(config) {
-  return sha256(canonicalJson(config));
+function balancedFrom(s, start) {
+  const open = s[start];
+  const close = open === "{" ? "}" : "]";
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+  for (let i = start; i < s.length; i++) {
+    const ch = s[i];
+    if (inString) {
+      if (escaped) escaped = false;
+      else if (ch === "\\") escaped = true;
+      else if (ch === '"') inString = false;
+      continue;
+    }
+    if (ch === '"') {
+      inString = true;
+    } else if (ch === open) {
+      depth++;
+    } else if (ch === close) {
+      depth--;
+      if (depth === 0) return s.slice(start, i + 1);
+    }
+  }
+  return null;
 }
-var import_node_crypto;
-var init_hash = __esm({
-  "src/util/hash.ts"() {
+var init_json_extract = __esm({
+  "src/assertions/json-extract.ts"() {
     "use strict";
     init_cjs_shims();
-    import_node_crypto = require("crypto");
-    init_canonical_json();
   }
 });
 
@@ -1157,80 +1291,6 @@ var init_contains = __esm({
   }
 });
 
-// src/assertions/json-extract.ts
-function extractJson(output, mode) {
-  if (mode === "strict") {
-    return tryParse(output.trim());
-  }
-  const fenced = extractFencedBlock(output);
-  if (fenced !== null) {
-    const r = tryParse(fenced);
-    if (r.ok) return r;
-  }
-  for (const run of balancedRuns(output)) {
-    const r = tryParse(run);
-    if (r.ok) return r;
-  }
-  return { ok: false };
-}
-function tryParse(s) {
-  try {
-    return { ok: true, value: JSON.parse(s) };
-  } catch {
-    return { ok: false };
-  }
-}
-function extractFencedBlock(s) {
-  const m = s.match(/```(?:json)?\s*\n?([\s\S]*?)```/i);
-  return m?.[1]?.trim() ?? null;
-}
-function* balancedRuns(s) {
-  let i = 0;
-  while (i < s.length) {
-    const rel = s.slice(i).search(/[{[]/);
-    if (rel === -1) return;
-    const start = i + rel;
-    const run = balancedFrom(s, start);
-    if (run === null) {
-      i = start + 1;
-    } else {
-      yield run;
-      i = start + run.length;
-    }
-  }
-}
-function balancedFrom(s, start) {
-  const open = s[start];
-  const close = open === "{" ? "}" : "]";
-  let depth = 0;
-  let inString = false;
-  let escaped = false;
-  for (let i = start; i < s.length; i++) {
-    const ch = s[i];
-    if (inString) {
-      if (escaped) escaped = false;
-      else if (ch === "\\") escaped = true;
-      else if (ch === '"') inString = false;
-      continue;
-    }
-    if (ch === '"') {
-      inString = true;
-    } else if (ch === open) {
-      depth++;
-    } else if (ch === close) {
-      depth--;
-      if (depth === 0) return s.slice(start, i + 1);
-    }
-  }
-  return null;
-}
-var init_json_extract = __esm({
-  "src/assertions/json-extract.ts"() {
-    "use strict";
-    init_cjs_shims();
-  }
-});
-
 // src/assertions/json-schema.ts
 var import_ajv, jsonSchema;
 var init_json_schema = __esm({
@@ -1692,6 +1752,11 @@ var init_results = __esm({
 });
 
 // src/orchestrator/run.ts
+function answerParses(output, assertions) {
+  return assertions.every(
+    (a) => a.type !== "json-schema" || extractJson(output, a.extract ?? "auto").ok
+  );
+}
 function resolvePrompt(template, vars) {
   return template.replace(/\{\{\s*(\w+)\s*\}\}/g, (_match, key) => {
     if (!(key in vars)) {
@@ -1719,17 +1784,35 @@ async function runRow(p) {
     const usage3 = { ...est, cost_usd: p.provider.estimateCost(est).cost_usd };
     return { vars: p.row.vars, output: null, cached: false, usage: usage3, assertions: [] };
   }
-  const key = cacheKey({
-    provider: p.providerConfig.type,
-    model: p.providerConfig.model,
-    temperature: temperature ?? 0,
-    max_tokens: maxTokens,
-    prompt_resolved: prompt5
-  });
+  const key = cacheKey(
+    {
+      provider: p.providerConfig.type,
+      model: p.providerConfig.model,
+      temperature: temperature ?? 0,
+      max_tokens: maxTokens,
+      prompt_resolved: prompt5
+    },
+    p.policyVersion
+  );
   let output;
   let usage2;
   let cached = false;
-  const hit = p.noCache ? void 0 : p.cache.get(key);
+  let cacheSaved;
+  let hit = p.noCache ? void 0 : p.cache.get(key, p.cacheMaxAgeMs !== void 0 ? { maxAgeMs: p.cacheMaxAgeMs } : {});
+  if (hit !== void 0 && !answerParses(hit.output, p.assertions)) {
+    if (p.cacheStrict !== true) p.cache.delete(key);
+    hit = void 0;
+  }
+  if (hit === void 0 && p.cacheStrict === true) {
+    return {
+      vars: p.row.vars,
+      output: null,
+      cached: false,
+      latency_ms: p.now() - t0,
+      error: { code: CACHE_MISS_CODE, message: "no usable cache entry (--cache-strict)" },
+      assertions: []
+    };
+  }
   if (hit !== void 0) {
     output = hit.output;
     cached = true;
@@ -1738,6 +1821,11 @@ async function runRow(p) {
       output_tokens: hit.usage.output_tokens,
       cost_usd: 0
     };
+    cacheSaved = {
+      input_tokens: hit.usage.input_tokens,
+      output_tokens: hit.usage.output_tokens,
+      cost_usd: p.provider.estimateCost(hit.usage).cost_usd
+    };
   } else {
     try {
       const res = await p.provider.complete({
@@ -1755,7 +1843,9 @@ async function runRow(p) {
         // lets a config pricing override win, else falls back to the table estimate (ADR-PRC01).
         cost_usd: p.provider.estimateCost(res.usage, res.cost_usd).cost_usd
       };
-      if (!p.noCache) p.cache.set(key, res);
+      if (!p.noCache && isCacheable(res) && answerParses(res.output, p.assertions)) {
+        p.cache.set(key, res);
+      }
     } catch (err) {
       return {
         vars: p.row.vars,
@@ -1801,6 +1891,7 @@ async function runRow(p) {
     cached,
     usage: finalUsage,
     latency_ms: p.now() - t0,
+    ...cacheSaved !== void 0 ? { cache_saved: cacheSaved } : {},
     ...assertionError !== void 0 ? { error: assertionError } : {},
     assertions: records
   };
@@ -1835,19 +1926,41 @@ function summarize(evals, durationMs) {
     cost_usd: 0,
     duration_ms: durationMs
   };
+  const cache = {
+    from_cache: 0,
+    from_model: 0,
+    missed: 0,
+    errored: 0,
+    saved: { input_tokens: 0, output_tokens: 0, cost_usd: 0 }
+  };
   for (const ev of evals) {
     for (const row of ev.rows) {
       summary.total += 1;
       summary[rowStatus(row)] += 1;
       summary.cost_usd += row.usage?.cost_usd ?? 0;
+      if (row.cached) {
+        cache.from_cache += 1;
+        cache.saved.input_tokens += row.cache_saved?.input_tokens ?? 0;
+        cache.saved.output_tokens += row.cache_saved?.output_tokens ?? 0;
+        cache.saved.cost_usd += row.cache_saved?.cost_usd ?? 0;
+      } else if (row.error?.code === CACHE_MISS_CODE) cache.missed += 1;
+      else if (row.output !== null) cache.from_model += 1;
+      else if (row.error !== void 0) cache.errored += 1;
     }
   }
-  return summary;
+  return { ...summary, cache };
 }
 async function runOrchestration(config, options, deps) {
+  const noCache = options.noCache ?? false;
+  const cacheStrict = options.cacheStrict ?? false;
+  if (noCache && cacheStrict) {
+    throw new RunConfigError("--no-cache and --cache-strict cannot be used together");
+  }
+  if (noCache && options.cacheMaxAgeMs !== void 0) {
+    throw new RunConfigError("--no-cache and --cache-max-age cannot be used together");
+  }
   const started = deps.now();
   const dryRun = options.dryRun ?? false;
-  const noCache = options.noCache ?? false;
   const evalResults = [];
   const prepared = selectEvals(config.evals, options.only).map((ev) => {
     const template = resolveTemplate(ev, deps.baseDir);
@@ -1874,7 +1987,9 @@ async function runOrchestration(config, options, deps) {
         cache: deps.cache,
         now: deps.now,
         dryRun,
-        noCache
+        noCache,
+        cacheStrict,
+        ...options.cacheMaxAgeMs !== void 0 ? { cacheMaxAgeMs: options.cacheMaxAgeMs } : {}
       })
     );
     const passed = rowResults.every((r) => rowStatus(r) === "passed");
@@ -1893,7 +2008,7 @@ async function runOrchestration(config, options, deps) {
     evals: evalResults
   };
 }
-var fs3, path3, DEFAULT_MAX_TOKENS2, PLUNE_VERSION, RunConfigError;
+var fs3, path3, DEFAULT_MAX_TOKENS2, PLUNE_VERSION, RunConfigError, CACHE_MISS_CODE;
 var init_run = __esm({
   "src/orchestrator/run.ts"() {
     "use strict";
@@ -1901,6 +2016,8 @@ var init_run = __esm({
     fs3 = __toESM(require("fs"), 1);
     path3 = __toESM(require("path"), 1);
     init_hash();
+    init_json_extract();
+    init_cache();
     init_assertions();
     init_judge();
     init_concurrency();
@@ -1915,20 +2032,29 @@ var init_run = __esm({
         this.name = "RunConfigError";
       }
     };
+    CACHE_MISS_CODE = "CACHE_MISS";
   }
 });
 
 // src/orchestrator/exit-code.ts
+function cacheMissRows(result) {
+  return result.evals.flatMap(
+    (ev) => ev.rows.filter((r) => r.error?.code === CACHE_MISS_CODE).map((r) => `${ev.id} ${JSON.stringify(r.vars)}`)
+  );
+}
 function exitCodeFor(result) {
   const { failed, errored } = result.summary;
   if (failed > 0) return 1;
   if (errored > 0) return 2;
   return 0;
 }
+var CACHE_STRICT_EXIT;
 var init_exit_code = __esm({
   "src/orchestrator/exit-code.ts"() {
     "use strict";
     init_cjs_shims();
+    init_run();
+    CACHE_STRICT_EXIT = 5;
   }
 });
 
@@ -1953,8 +2079,11 @@ var init_dataset = __esm({
 // src/orchestrator/index.ts
 var orchestrator_exports = {};
 __export(orchestrator_exports, {
+  CACHE_MISS_CODE: () => CACHE_MISS_CODE,
+  CACHE_STRICT_EXIT: () => CACHE_STRICT_EXIT,
   RunConfigError: () => RunConfigError,
   buildJudge: () => buildJudge,
+  cacheMissRows: () => cacheMissRows,
   exitCodeFor: () => exitCodeFor,
   loadDataset: () => loadDataset,
   mapLimit: () => mapLimit,
@@ -2013,6 +2142,11 @@ async function handleRun(options, depsFactory) {
   const factory = depsFactory ?? ((c, d) => buildRealDeps(c, d, options.dryRun));
   const deps = factory(config, baseDir);
   try {
+    const stale = deps.cache.staleEntries ?? 0;
+    if (stale > 0) {
+      process.stderr.write(`cache policy changed, ${stale} entries are stale (dropped)
+`);
+    }
     const result = await runOrchestration(
       config,
       {
@@ -2020,7 +2154,9 @@ async function handleRun(options, depsFactory) {
         ...options.only !== void 0 ? { only: options.only } : {},
         ...options.concurrency !== void 0 ? { concurrency: options.concurrency } : {},
         ...options.noCache !== void 0 ? { noCache: options.noCache } : {},
-        ...options.bail !== void 0 ? { bail: options.bail } : {}
+        ...options.bail !== void 0 ? { bail: options.bail } : {},
+        ...options.cacheStrict === true ? { cacheStrict: true } : {},
+        ...options.cacheMaxAgeMs !== void 0 ? { cacheMaxAgeMs: options.cacheMaxAgeMs } : {}
       },
       deps
     );
@@ -2045,6 +2181,7 @@ var init_run2 = __esm({
     init_cache2();
     init_orchestrator();
     NOOP_CACHE = { get: () => void 0, set: () => {
+    }, delete: () => {
     }, clear: () => {
     }, close: () => {
     } };
@@ -2104,6 +2241,15 @@ function renderConsole(result, opts = {}) {
   lines.push(
     `${passedTxt} \xB7 ${failedTxt} \xB7 ${erroredTxt}${blockedTxt}${skippedTxt} \xB7 ${summary.total} total \xB7 $${summary.cost_usd.toFixed(4)} \xB7 ${summary.duration_ms}ms`
   );
+  const c = summary.cache;
+  if (c !== void 0 && c.from_cache + c.from_model + c.missed > 0) {
+    const missedTxt = c.missed > 0 ? ` \xB7 ${s.yellow(`${c.missed} missed`)}` : "";
+    lines.push(
+      s.dim(
+        `cache: ${c.from_cache} from cache \xB7 ${c.from_model} from model${missedTxt} \xB7 saved ${c.saved.input_tokens + c.saved.output_tokens} tokens, $${c.saved.cost_usd.toFixed(4)}`
+      )
+    );
+  }
   lines.push("");
   for (const ev of result.evals) {
     const mark = ev.passed ? s.green("PASS") : s.red("FAIL");
@@ -3903,7 +4049,7 @@ async function handleRunStart(options = {}) {
     kind: options.kind ?? "automated",
     externalKey,
     // The same name the reporter would give it; `PLUNE_RUN_TITLE` replaces it, as everywhere.
-    title: env.config.title ?? defaultRunTitle()
+    title: options.name ?? env.config.title ?? defaultRunTitle()
   });
   if (!out.ok) throw new RunCommandError(`Could not start the run \u2014 ${out.detail || out.kind}.`);
   const result = {
@@ -4122,6 +4268,7 @@ var init_run_result_schema = __esm({
       latency_ms: import_zod3.z.number().optional(),
       error: runErrorSchema.optional(),
       assertions: import_zod3.z.array(assertionResultRecordSchema2),
+      cache_saved: import_zod3.z.object({ input_tokens: import_zod3.z.number(), output_tokens: import_zod3.z.number(), cost_usd: import_zod3.z.number() }).optional(),
       /** ADR 0045. Optional and additive — the same enum the platform validates on `POST /v1/runs`. */
       inconclusive: import_zod3.z.enum(ROW_INCONCLUSIVE).optional()
     });
@@ -4134,7 +4281,14 @@ var init_run_result_schema = __esm({
       blocked: import_zod3.z.number().optional(),
       skipped: import_zod3.z.number().optional(),
       cost_usd: import_zod3.z.number(),
-      duration_ms: import_zod3.z.number()
+      duration_ms: import_zod3.z.number(),
+      cache: import_zod3.z.object({
+        from_cache: import_zod3.z.number(),
+        from_model: import_zod3.z.number(),
+        missed: import_zod3.z.number(),
+        errored: import_zod3.z.number(),
+        saved: import_zod3.z.object({ input_tokens: import_zod3.z.number(), output_tokens: import_zod3.z.number(), cost_usd: import_zod3.z.number() })
+      }).optional()
     });
     evalResultSchema = import_zod3.z.object({
       id: import_zod3.z.string(),
@@ -4309,6 +4463,7 @@ __export(plan_exports, {
   grepFragment: () => grepFragment,
   handlePlanGrep: () => handlePlanGrep,
   handleRunGrep: () => handleRunGrep,
+  platformGet: () => platformGet,
   runGrepSummary: () => runGrepSummary
 });
 function grepFragment(c) {
@@ -5420,6 +5575,7 @@ async function handleRunImport(options) {
         token,
         ...options.fetchImpl !== void 0 ? { fetchImpl: options.fetchImpl } : {},
         ...options.key !== void 0 ? { externalKey: options.key } : {},
+        ...options.name !== void 0 ? { title: options.name } : {},
         ...options.create === true ? { offerDiscovered: true } : {},
         ...options.uploadConcurrency !== void 0 ? { uploadConcurrency: options.uploadConcurrency } : {},
         kind: "automated",
@@ -6645,6 +6801,130 @@ var init_requirements_import = __esm({
   }
 });
 
+// src/cli/commands/verify.ts
+var verify_exports = {};
+__export(verify_exports, {
+  compare: () => compare,
+  formatVerify: () => formatVerify,
+  handleVerify: () => handleVerify,
+  testsOfListing: () => testsOfListing
+});
+function testsOfListing(listing) {
+  const found = /* @__PURE__ */ new Map();
+  const walk2 = (suite, titles) => {
+    for (const spec of suite.specs ?? []) {
+      const annotated = spec.tests?.flatMap((t) => t.annotations ?? []).find(
+        (a) => a.type === "PluneId" && a.description !== void 0 && a.description !== ""
+      )?.description;
+      const id = annotated ?? TOKEN4.exec(spec.title)?.[1];
+      const pathTitle = [...titles, spec.title].join("#");
+      if (!found.has(pathTitle))
+        found.set(pathTitle, { pathTitle, ...id === void 0 ? {} : { id } });
+    }
+    for (const child of suite.suites ?? []) walk2(child, [...titles, child.title]);
+  };
+  for (const file of listing.suites ?? []) walk2(file, [file.title.replace(/\\/g, "/")]);
+  return [...found.values()];
+}
+function compare(code, cases) {
+  const live = cases.filter((c) => c.type === "automated" && c.state !== "deprecated");
+  const byPath = /* @__PURE__ */ new Map();
+  for (const c of live)
+    for (const k of c.externalKeys ?? []) if (k.kind === "path-title") byPath.set(k.value, c);
+  const byId = new Map(live.map((c) => [c.id, c]));
+  const seen = /* @__PURE__ */ new Set();
+  const onlyInCode = [];
+  for (const t of code) {
+    const hit = (t.id === void 0 ? void 0 : byId.get(t.id)) ?? byPath.get(t.pathTitle);
+    if (hit === void 0) onlyInCode.push({ pathTitle: t.pathTitle });
+    else seen.add(hit.id);
+  }
+  return {
+    matched: seen.size,
+    onlyInPlatform: live.filter((c) => !seen.has(c.id)).map((c) => ({ id: c.id, title: c.title, state: c.state, detached: c.state === "detached" })),
+    onlyInCode
+  };
+}
+function runPlaywrightList(args, cwd) {
+  try {
+    return (0, import_node_child_process3.execFileSync)("npx", ["playwright", "test", ...args], {
+      cwd,
+      encoding: "utf8",
+      maxBuffer: 256 * 1024 * 1024,
+      stdio: ["ignore", "pipe", "pipe"],
+      shell: process.platform === "win32"
+    });
+  } catch (err) {
+    const stderr = err.stderr?.trim();
+    throw new SyncFileError(`could not list the Playwright tests${stderr ? `: ${stderr}` : ""}`);
+  }
+}
+async function handleVerify(deps) {
+  const get = platformGet(deps);
+  let raw;
+  if (deps.from !== void 0) {
+    try {
+      raw = (0, import_node_fs8.readFileSync)(deps.from, "utf8");
+    } catch {
+      throw new SyncFileError(`cannot read ${deps.from}`);
+    }
+  } else {
+    const args = [
+      "--list",
+      "--reporter=json",
+      ...deps.playwrightConfig === void 0 ? [] : ["--config", deps.playwrightConfig]
+    ];
+    raw = (deps.list ?? runPlaywrightList)(args, deps.cwd ?? process.cwd());
+  }
+  let listing;
+  try {
+    listing = JSON.parse(raw);
+  } catch {
+    throw new SyncFileError("the Playwright listing is not JSON");
+  }
+  const cases = [];
+  let cursor = null;
+  do {
+    const page = await get(
+      `/v1/test-cases?type=automated&stability=0&limit=200${cursor === null ? "" : `&cursor=${cursor}`}`
+    );
+    cases.push(...page.testCases);
+    cursor = page.nextCursor;
+  } while (cursor !== null);
+  return compare(testsOfListing(listing), cases);
+}
+function formatVerify(r) {
+  const lines = [];
+  if (r.onlyInPlatform.length > 0) {
+    lines.push(`In Plune, not in the code (${r.onlyInPlatform.length}):`);
+    for (const c of r.onlyInPlatform) {
+      lines.push(
+        `  ${c.id}  ${c.title}  [${c.detached ? "already detached" : `${c.state} \u2014 a full run would detach it`}]`
+      );
+    }
+  }
+  if (r.onlyInCode.length > 0) {
+    lines.push(`In the code, not in Plune (${r.onlyInCode.length}):`);
+    for (const t of r.onlyInCode) lines.push(`  ${t.pathTitle}`);
+  }
+  lines.push(
+    r.onlyInPlatform.length + r.onlyInCode.length === 0 ? `No drift: ${r.matched} case(s) match the code.` : `${r.matched} matched. Nothing was changed.`
+  );
+  return lines.join("\n") + "\n";
+}
+var import_node_child_process3, import_node_fs8, TOKEN4;
+var init_verify = __esm({
+  "src/cli/commands/verify.ts"() {
+    "use strict";
+    init_cjs_shims();
+    import_node_child_process3 = require("child_process");
+    import_node_fs8 = require("fs");
+    init_plan();
+    init_sync();
+    TOKEN4 = /@P([A-Za-z0-9_-]+)/;
+  }
+});
+
 // src/cli.ts
 var cli_exports = {};
 __export(cli_exports, {
@@ -6654,11 +6934,25 @@ module.exports = __toCommonJS(cli_exports);
 init_cjs_shims();
 init_api_url();
 var import_commander = require("commander");
-var import_node_fs8 = require("fs");
+var import_node_fs9 = require("fs");
 var import_node_path5 = require("path");
 var import_node_url = require("url");
+
+// src/util/duration.ts
+init_cjs_shims();
+var UNIT_MS = { ms: 1, s: 1e3, m: 6e4, h: 36e5, d: 864e5 };
+function parseDuration(text2) {
+  const m = /^(\d+(?:\.\d+)?)(ms|s|m|h|d)?$/.exec(text2.trim());
+  const ms = m === null ? NaN : Number(m[1]) * UNIT_MS[m[2] ?? "ms"];
+  if (!Number.isFinite(ms) || ms <= 0) {
+    throw new Error(`Invalid duration "${text2}" (use e.g. 90s, 15m, 12h, 7d)`);
+  }
+  return ms;
+}
+
+// src/cli.ts
 var __dirname = (0, import_node_path5.dirname)((0, import_node_url.fileURLToPath)(importMetaUrl));
-var pkg = JSON.parse((0, import_node_fs8.readFileSync)((0, import_node_path5.join)(__dirname, "..", "package.json"), "utf-8"));
+var pkg = JSON.parse((0, import_node_fs9.readFileSync)((0, import_node_path5.join)(__dirname, "..", "package.json"), "utf-8"));
 function toFormat(s) {
   return s === "json" || s === "markdown" ? s : "console";
 }
@@ -6667,7 +6961,7 @@ function shouldColor(outputPath) {
 }
 function writeReport(text2, outputPath) {
   if (outputPath !== void 0) {
-    (0, import_node_fs8.writeFileSync)(outputPath, text2);
+    (0, import_node_fs9.writeFileSync)(outputPath, text2);
   } else {
     process.stdout.write(text2 + "\n");
   }
@@ -6720,13 +7014,17 @@ function createProgram() {
     "Run a subset by eval id or tag (repeatable)",
     (val, prev) => [...prev, val],
     []
-  ).option("--concurrency <n>", "Override provider concurrency", (v) => parseInt(v, 10)).option("--no-cache", "Bypass the local cache for this run").option("--bail", "Stop after the first failing eval", false).option("--format <fmt>", "console | json | markdown", "console").option("-o, --output <path>", "Write the report to a file instead of stdout").action(
+  ).option("--concurrency <n>", "Override provider concurrency", (v) => parseInt(v, 10)).option("--no-cache", "Bypass the local cache for this run").option(
+    "--cache-strict",
+    "Every row must be in the cache: no model calls for completions; a miss ends the run with exit 5 and lists the rows",
+    false
+  ).option("--cache-max-age <duration>", "Treat cache entries older than this as a miss (e.g. 90s, 15m, 12h, 7d)").option("--bail", "Stop after the first failing eval", false).option("--format <fmt>", "console | json | markdown", "console").option("-o, --output <path>", "Write the report to a file instead of stdout").action(
     async (options, command) => {
       const globals = command.optsWithGlobals();
       const configPath = globals.config;
       const [
         { handleRun: handleRun2 },
-        { exitCodeFor: exitCodeFor2, RunConfigError: RunConfigError2 },
+        { exitCodeFor: exitCodeFor2, cacheMissRows: cacheMissRows2, CACHE_STRICT_EXIT: CACHE_STRICT_EXIT2, RunConfigError: RunConfigError2 },
         { renderReport: renderReport2 },
         { ConfigNotFoundError: ConfigNotFoundError2, ConfigValidationError: ConfigValidationError2, YamlParseError: YamlParseError2 },
         { AuthError: AuthError2 }
@@ -6738,13 +7036,36 @@ function createProgram() {
         Promise.resolve().then(() => (init_errors2(), errors_exports2))
       ]);
       try {
+        if (options.cache === false && options.cacheMaxAge !== void 0) {
+          process.stderr.write("--no-cache and --cache-max-age cannot be used together\n");
+          process.exit(2);
+          return;
+        }
+        if (options.cache === false && options.cacheStrict) {
+          process.stderr.write("--no-cache and --cache-strict cannot be used together\n");
+          process.exit(2);
+          return;
+        }
+        let cacheMaxAgeMs;
+        if (options.cacheMaxAge !== void 0) {
+          try {
+            cacheMaxAgeMs = parseDuration(options.cacheMaxAge);
+          } catch (err) {
+            process.stderr.write(`--cache-max-age: ${err.message}
+`);
+            process.exit(2);
+            return;
+          }
+        }
         const result = await handleRun2({
           dryRun: options.dryRun,
           ...configPath !== void 0 ? { configPath } : {},
           ...options.only.length > 0 ? { only: options.only } : {},
           ...typeof options.concurrency === "number" && Number.isFinite(options.concurrency) ? { concurrency: options.concurrency } : {},
           noCache: options.cache === false,
-          bail: options.bail
+          bail: options.bail,
+          ...options.cacheStrict ? { cacheStrict: true } : {},
+          ...cacheMaxAgeMs !== void 0 ? { cacheMaxAgeMs } : {}
         });
         if (options.only.length > 0 && result.evals.length === 0) {
           process.stderr.write("No evals matched the selector.\n");
@@ -6754,6 +7075,15 @@ function createProgram() {
         });
         writeReport(text2, options.output);
         process.exitCode = exitCodeFor2(result);
+        const missed = cacheMissRows2(result);
+        if (missed.length > 0) {
+          process.stderr.write(
+            `--cache-strict: ${missed.length} row(s) have no cache entry:
+${missed.map((m) => `  - ${m}`).join("\n")}
+`
+          );
+          process.exitCode = CACHE_STRICT_EXIT2;
+        }
       } catch (err) {
         const verbose = globals.verbose === true;
         if (err instanceof ConfigValidationError2) {
@@ -6803,11 +7133,12 @@ function createProgram() {
   };
   const verboseOf = (command) => command.optsWithGlobals().verbose === true;
   const FAIL_ON_FAILURES = "Exit with the platform's verdict on the run: 1 on a failed or broken result (muted ones do not count), on an expected case that never ran, or when there is no verdict at all";
-  runCommand.command("start").description("Open a platform run (or join one already carrying the key) and print its id").option("--key <externalKey>", "The key several processes share; generated when absent").option("--json", "Print one machine-readable line instead of prose", false).action(async (options, command) => {
+  runCommand.command("start").description("Open a platform run (or join one already carrying the key) and print its id").option("--key <externalKey>", "The key several processes share; generated when absent").option("--name <text>", "What to call the run in the list (default: the folder name, the time and where it ran; PLUNE_RUN_TITLE sets it too)").option("--json", "Print one machine-readable line instead of prose", false).action(async (options, command) => {
     try {
       const { handleRunStart: handleRunStart2 } = await Promise.resolve().then(() => (init_run_lifecycle(), run_lifecycle_exports));
       await handleRunStart2({
         ...options.key !== void 0 ? { key: options.key } : {},
+        ...options.name !== void 0 ? { name: options.name } : {},
         json: options.json
       });
     } catch (err) {
@@ -6831,12 +7162,13 @@ function createProgram() {
   );
   runCommand.command("exec").argument("<command...>", "The command to run inside the run, after `--`").description(
     "Open a run, run a command inside it, close the run \u2014 whatever the command returns"
-  ).option("--key <externalKey>", "The key several processes share; generated when absent").option("--fail-on-failures", `${FAIL_ON_FAILURES}; the command's own exit code when no result reached the run`, false).action(async (argv, options, command) => {
+  ).option("--key <externalKey>", "The key several processes share; generated when absent").option("--name <text>", "What to call the run in the list (default: the folder name, the time and where it ran; PLUNE_RUN_TITLE sets it too)").option("--fail-on-failures", `${FAIL_ON_FAILURES}; the command's own exit code when no result reached the run`, false).action(async (argv, options, command) => {
     try {
       const { handleRunExec: handleRunExec2 } = await Promise.resolve().then(() => (init_run_lifecycle(), run_lifecycle_exports));
       process.exitCode = await handleRunExec2({
         argv,
         ...options.key !== void 0 ? { key: options.key } : {},
+        ...options.name !== void 0 ? { name: options.name } : {},
         ...options.failOnFailures ? { failOnFailures: true } : {}
       });
     } catch (err) {
@@ -6888,7 +7220,7 @@ function createProgram() {
       await failRunCommand(err, verboseOf(command));
     }
   });
-  runCommand.command("import").argument("<file>", "A JUnit XML or Playwright JSON report, or an Allure allure-results folder").description("Turn a report another runner wrote into a run in Plune \u2014 no provider key needed").option("--format <fmt>", "junit | playwright-json | allure; detected from the file (a folder is allure) when omitted").option("--key <externalKey>", "The key several jobs share so their reports land in one run").option("--create", "Offer tests nothing matched to the review queue", false).option(
+  runCommand.command("import").argument("<file>", "A JUnit XML or Playwright JSON report, or an Allure allure-results folder").description("Turn a report another runner wrote into a run in Plune \u2014 no provider key needed").option("--format <fmt>", "junit | playwright-json | allure; detected from the file (a folder is allure) when omitted").option("--key <externalKey>", "The key several jobs share so their reports land in one run").option("--name <text>", "What to call the run in the list (default: the folder name, the time and where it ran; PLUNE_RUN_TITLE sets it too)").option("--create", "Offer tests nothing matched to the review queue", false).option(
     "--fail-on-missing-attachments",
     "Exit 1 when a file the report names ([[ATTACHMENT|path]] in JUnit, a source in allure-results) is not there; the results are sent either way",
     false
@@ -6930,6 +7262,7 @@ function createProgram() {
           file,
           ...format !== void 0 ? { format } : {},
           ...options.key !== void 0 ? { key: options.key } : {},
+          ...options.name !== void 0 ? { name: options.name } : {},
           ...options.create ? { create: true } : {},
           ...options.uploadConcurrency !== void 0 ? { uploadConcurrency: options.uploadConcurrency } : {},
           ...options.timeout !== void 0 ? { timeoutSec: options.timeout } : {},
@@ -7190,6 +7523,31 @@ function createProgram() {
       failUnexpected(err, verbose);
     }
   });
+  program.command("verify").description(
+    "Compare the project's automated cases with the Playwright tests in the code, read-only \u2014 both sides of the drift, nothing written"
+  ).option("--from <file>", "A saved `playwright test --list --reporter=json` document instead of running Playwright").option("--playwright-config <path>", "Passed to `playwright test --config`").option("--json", "Print the report as JSON", false).option("--fail-on-drift", "Exit 1 when either side has something the other lacks", false).action(
+    async (options, command) => {
+      const verbose = verboseOf(command);
+      const { handleVerify: handleVerify2, formatVerify: formatVerify2 } = await Promise.resolve().then(() => (init_verify(), verify_exports));
+      const { reportCasesFailure: reportCasesFailure2 } = await Promise.resolve().then(() => (init_cases(), cases_exports));
+      try {
+        const report = await handleVerify2({
+          ...options.from === void 0 ? {} : { from: options.from },
+          ...options.playwrightConfig === void 0 ? {} : { playwrightConfig: options.playwrightConfig }
+        });
+        process.stdout.write(options.json ? JSON.stringify(report, null, 2) + "\n" : formatVerify2(report));
+        if (options.failOnDrift && report.onlyInPlatform.length + report.onlyInCode.length > 0) process.exitCode = 1;
+      } catch (err) {
+        const code = reportCasesFailure2(err, (s) => process.stderr.write(s));
+        if (code !== null) {
+          maybeStack(err, verbose);
+          process.exitCode = code;
+          return;
+        }
+        failUnexpected(err, verbose);
+      }
+    }
+  );
   program.on("command:*", (operands) => {
     process.stderr.write(`Unknown command: ${operands.join(" ")}
 `);
@@ -7201,7 +7559,7 @@ function isMainModule() {
   const entry = process.argv[1];
   if (entry === void 0) return false;
   try {
-    return (0, import_node_fs8.realpathSync)(entry) === (0, import_node_fs8.realpathSync)((0, import_node_url.fileURLToPath)(importMetaUrl));
+    return (0, import_node_fs9.realpathSync)(entry) === (0, import_node_fs9.realpathSync)((0, import_node_url.fileURLToPath)(importMetaUrl));
   } catch {
     return false;
   }