@remnic/bench 9.69.61 → 9.69.64

package/dist/index.d.ts

@@ -5825,6 +5825,12 @@ interface InjectionSuiteRunMetadata {
     model: string;
     baseUrl: string;
     requestTimeoutMs: number;
+    /** Resolved OpenAI-compatible backend; it selects non-generic request fields (#3078). */
+    backend?: string;
+    /** Row count the grid WOULD have had before `--limit` sliced it (#3080): lets an analyzer tell a truncating limit from a no-op one. */
+    unslicedPlannedRows?: number;
+    /** Attacker endpoint, persisted for resume-hash recomputation (PR #3081 r3). */
+    attackerBaseUrl?: string;
     stage: InjectionSuiteStage;
     runKind: "dev" | "pilot" | "main";
     modelProfileHash: string;
@@ -5904,6 +5910,7 @@ declare function injectionSuiteResumeContractHash(metadata: {
     model: string;
     baseUrl: string;
     requestTimeoutMs: number;
+    backend?: string;
     stage?: string;
     runKind?: string;
     modelProfileHash?: string;
@@ -6217,6 +6224,8 @@ interface OnlineAdaptiveStatistics {
         truncatedChains?: number;
         /** Episode rows whose key is not in the frozen design; dropped from scoring. */
         unexpectedRows?: number;
+        /** The run recorded a `--limit`, so its design is a subset of the registered grid. */
+        limitedDesign?: boolean;
         /** Lines recorded in online-corpus.jsonl (one per attacker iteration). */
         corpusLines?: number;
         /** Was an online-corpus-manifest.json written? */

package/dist/index.js

@@ -47900,6 +47900,37 @@ function readToolCalls(value) {
     ];
   });
 }
+var OPENAI_COMPAT_BACKEND_ENV = "REMNIC_BENCH_OPENAI_COMPAT_BACKEND";
+function openAiCompatBackend(baseUrl, override) {
+  const named = (override ?? process.env[OPENAI_COMPAT_BACKEND_ENV] ?? "").trim().toLowerCase();
+  if (named === "nim" || named === "ollama" || named === "generic") return named;
+  if (named.length > 0) {
+    throw new InjectionSuiteHostFault(
+      `${OPENAI_COMPAT_BACKEND_ENV} must be one of nim, ollama, generic (got ${named})`
+    );
+  }
+  let host;
+  try {
+    host = new URL(trimTrailingSlashes(baseUrl)).hostname.toLowerCase().replace(/\.+$/, "");
+  } catch {
+    return "generic";
+  }
+  if (host === "integrate.api.nvidia.com" || host.endsWith(".api.nvidia.com")) return "nim";
+  return "generic";
+}
+function openAiCompatExtensions(baseUrl, model, backendOverride) {
+  const backend = openAiCompatBackend(baseUrl, backendOverride);
+  if (backend === "generic") return {};
+  const extensions = {};
+  const lowEffortModel = model.startsWith("openai/gpt-oss-") || model === "meta/llama-3.2-11b-vision-instruct";
+  const thinkingFamily = /qwen|nemotron|deepseek/i.test(model);
+  if (lowEffortModel) extensions.reasoning_effort = "low";
+  else if (thinkingFamily) extensions.reasoning_effort = "none";
+  if (backend === "nim" && /qwen|nemotron/i.test(model)) {
+    extensions.chat_template_kwargs = { enable_thinking: false };
+  }
+  return extensions;
+}
 async function completeChatResult(options, prompt) {
   const messages = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
   const timeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
@@ -47943,15 +47974,14 @@ async function completeChatResult(options, prompt) {
         temperature: 0,
         ...options.seed !== void 0 ? { seed: options.seed } : {},
         max_tokens: 256,
-        // `reasoning_effort` is an OpenAI reasoning-model parameter that NIM
-        // and Ollama accept; strict generic endpoints (LM Studio) reject
-        // unknown fields. Send it only to the model families that use it.
-        ...model.startsWith("openai/gpt-oss-") || model === "meta/llama-3.2-11b-vision-instruct" ? { reasoning_effort: "low" } : /qwen|nemotron|deepseek/i.test(model) ? { reasoning_effort: "none" } : {},
-        // `chat_template_kwargs` is a vLLM/NIM extension, not part of the
-        // Chat Completions contract; strict endpoints (api.openai.com, the
-        // NVIDIA Llama 3.2 endpoint) reject it with HTTP 400. Send it only
-        // to models whose chat template defines `enable_thinking`.
-        .../qwen|nemotron/i.test(model) ? { chat_template_kwargs: { enable_thinking: false } } : {},
+        // Non-generic request fields are gated by BOTH the backend and the
+        // model family (#3078): an unknown OpenAI-compatible server (LM
+        // Studio, api.openai.com) rejects unknown fields with HTTP 400, and
+        // the bench runner would classify that as a host fault and retry
+        // forever. Unknown backends therefore receive the generic contract
+        // only. Gating is derived from the endpoint, not probed, so a frozen
+        // run's request shape is reproducible from `run.json`.
+        ...openAiCompatExtensions(base2, model),
         ...tools ? {
           tools,
           tool_choice: options.forceSafeTool ? { type: "function", function: { name: "safe_tool" } } : "auto"
@@ -48550,6 +48580,9 @@ function injectionSuiteResumeContractHash(metadata) {
       model: metadata.model,
       baseUrl: metadata.baseUrl,
       requestTimeoutMs: metadata.requestTimeoutMs,
+      // Folded in only when recorded, so a run frozen before the backend
+      // was part of the identity keeps its existing resume hash (#3079).
+      ...metadata.backend === void 0 ? {} : { backend: metadata.backend },
       stage: metadata.stage ?? "base",
       runKind: metadata.runKind ?? "dev",
       modelProfileHash: metadata.modelProfileHash ?? "",
@@ -48567,22 +48600,28 @@ function hostFaultRetryDelayMs(message, consecutiveFaults) {
 function resolvedExecutorContract(input) {
   const executor = input.executor ?? "local";
   if (executor === "local") {
-    return { executor, model: "", baseUrl: "", requestTimeoutMs: 0 };
+    return { executor, model: "", baseUrl: "", requestTimeoutMs: 0, backend: "none" };
   }
   const requestTimeoutMs = input.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
   if (executor === "openai-compat") {
+    const baseUrl = input.baseUrl ?? DEFAULT_OPENAI_COMPAT_BASE_URL;
     return {
       executor,
       model: input.model ?? DEFAULT_OLLAMA_MODEL,
-      baseUrl: input.baseUrl ?? DEFAULT_OPENAI_COMPAT_BASE_URL,
-      requestTimeoutMs
+      baseUrl,
+      requestTimeoutMs,
+      // The backend selects the non-generic request fields, so a run resumed
+      // under a different override is a different experimental condition and
+      // must not reuse checkpoints (#3078, PR #3079 post-cap).
+      backend: openAiCompatBackend(baseUrl)
     };
   }
   return {
     executor,
     model: input.model ?? DEFAULT_OLLAMA_MODEL,
     baseUrl: input.baseUrl ?? DEFAULT_OLLAMA_BASE_URL,
-    requestTimeoutMs
+    requestTimeoutMs,
+    backend: "native"
   };
 }
 function planInjectionSuiteRows(input) {
@@ -48767,6 +48806,7 @@ async function runInjectionSuiteCliCommand(input) {
     model: contract.model,
     baseUrl: contract.baseUrl,
     requestTimeoutMs: contract.requestTimeoutMs,
+    backend: contract.backend,
     stage: input.stage ?? "base",
     runKind: input.runKind ?? "dev",
     modelProfileHash: frozen.profile.modelProfileHash,
@@ -49628,6 +49668,44 @@ async function readCorpusManifest(runDir) {
   const hashVerified = corpusBytes.length > 0 && createHash28("sha256").update(corpusBytes).digest("hex") === manifest.corpusSha256;
   return { manifest, hashVerified };
 }
+var INJECTION_SUITE_ONLINE_RESUME_CONTRACT = "h5-injection-suite-online-resume-v1";
+function injectionSuiteResumeContractHashForOnline(metadata) {
+  return createHash28("sha256").update(
+    JSON.stringify({
+      contract: INJECTION_SUITE_ONLINE_RESUME_CONTRACT,
+      suiteVersion: metadata.suiteVersion,
+      modelProfileId: metadata.modelProfileId,
+      seeds: metadata.seeds,
+      variantsPerFamily: metadata.variantsPerFamily,
+      family: metadata.family ?? null,
+      limit: metadata.limit,
+      executor: metadata.executor,
+      model: metadata.model,
+      baseUrl: metadata.baseUrl,
+      requestTimeoutMs: metadata.requestTimeoutMs,
+      // Folded in only when recorded, so runs frozen before the backend
+      // became part of the identity keep their resume hash (#3079).
+      ...metadata.backend === void 0 ? {} : { backend: metadata.backend },
+      // The unsliced count decides whether a limit truncated the design,
+      // so it is tamper-evident: folded into the resume hash and verified
+      // by the analyzer whenever it is recorded (#3080, PR #3081 r2).
+      ...metadata.unslicedPlannedRows === void 0 ? {} : { unslicedPlannedRows: metadata.unslicedPlannedRows },
+      stage: metadata.stage ?? ONLINE_ADAPTIVE_STAGE,
+      runKind: metadata.runKind ?? "dev",
+      modelProfileHash: metadata.modelProfileHash ?? "",
+      corpusManifestHash: metadata.corpusManifestHash ?? "",
+      expectedDesignHash: metadata.expectedDesignHash ?? "",
+      decisionRuleHash: metadata.decisionRuleHash ?? "",
+      gitSha: metadata.gitSha ?? "",
+      attackerExecutor: metadata.attackerExecutor ?? "",
+      attackerModel: metadata.attackerModel ?? "",
+      attackerBaseUrl: metadata.attackerBaseUrl ?? "",
+      attackerModelDigest: metadata.attackerModelDigest ?? "",
+      attackerPromptSha256: metadata.attackerPromptSha256 ?? "",
+      attackerIterations: metadata.attackerIterations ?? 0
+    })
+  ).digest("hex");
+}
 async function analyzeInjectionSuiteOnlineAdaptiveRun(runDir) {
   const [metadataText, designText, episodeLines, corpus] = await Promise.all([
     readFile29(path44.join(runDir, "run.json"), "utf8"),
@@ -49738,7 +49816,19 @@ async function analyzeInjectionSuiteOnlineAdaptiveRun(runDir) {
   const truncatedChains = [...maxPlannedIteration.values()].filter(
     (maxIteration) => maxIteration < finalIteration
   ).length;
-  const incomplete = !corpusManifestPresent || !manifestHashVerified || missingPlannedRows > 0 || neverGeneratedIterations > 0 || truncatedChains > 0 || unexpectedRows > 0;
+  const recordedLimit = metadata.limit ?? 0;
+  const unslicedPresent = metadata.unslicedPlannedRows !== void 0;
+  const recordedUnsliced = unslicedPresent && Number.isInteger(metadata.unslicedPlannedRows) && (metadata.unslicedPlannedRows ?? 0) >= design.rows.length && (metadata.unslicedPlannedRows ?? 0) > 0 ? metadata.unslicedPlannedRows : void 0;
+  let unsliced = recordedUnsliced;
+  const newContract = metadata.attackerBaseUrl !== void 0;
+  const digest = metadata.attackerModelDigest ?? "";
+  const digestCandidates = digest === "unverified" ? [digest, ""] : [digest];
+  const mustVerify = newContract || unslicedPresent;
+  const resumeHashVerifies = !mustVerify || digestCandidates.some((candidate) => injectionSuiteResumeContractHashForOnline({ ...metadata, attackerModelDigest: candidate }) === metadata.resumeContractHash);
+  const unslicedUnverifiable = !resumeHashVerifies;
+  if (unslicedUnverifiable) unsliced = void 0;
+  const limitedDesign = Number.isInteger(recordedLimit) && recordedLimit > 0 && (unsliced === void 0 || recordedLimit < unsliced) || unslicedUnverifiable;
+  const incomplete = limitedDesign || !corpusManifestPresent || !manifestHashVerified || missingPlannedRows > 0 || neverGeneratedIterations > 0 || truncatedChains > 0 || unexpectedRows > 0;
   const statistics = analyzeInjectionSuiteOnlineAdaptiveRows({
     rows,
     clusterByVariantBase,
@@ -49761,6 +49851,7 @@ async function analyzeInjectionSuiteOnlineAdaptiveRun(runDir) {
     missingPlannedRows,
     neverGeneratedIterations,
     truncatedChains,
+    limitedDesign,
     unexpectedRows,
     corpusLines: corpusSeenCount,
     corpusManifestPresent,
@@ -50138,6 +50229,15 @@ async function runInjectionSuiteOnlineAdaptive(rawInput, deps = DEFAULT_ONLINE_D
     iterations: input.attackerIterations,
     ...input.limit === void 0 ? {} : { limit: input.limit }
   });
+  const unslicedPlannedRows = input.limit === void 0 ? void 0 : planOnlineAdaptiveRows({
+    seeds: input.seeds,
+    ...input.seedBase === void 0 ? {} : { seedBase: input.seedBase },
+    variantsPerFamily: input.variantsPerFamily,
+    modelProfileId: input.modelProfileId,
+    ...input.family === void 0 ? {} : { family: input.family },
+    ...input.arms?.length ? { arms: input.arms } : {},
+    iterations: input.attackerIterations
+  }).length;
   const seeds = [...new Set(planned.map((row) => row.seed))];
   const frozen = prepareInjectionSuiteFreeze(input, planned);
   const defendedContract = resolvedExecutorContract({ ...input, executor: input.executor ?? "openai-compat" });
@@ -50148,10 +50248,12 @@ async function runInjectionSuiteOnlineAdaptive(rawInput, deps = DEFAULT_ONLINE_D
     variantsPerFamily: input.variantsPerFamily,
     family: input.family ?? null,
     limit: input.limit ?? null,
+    ...input.limit === void 0 ? {} : { unslicedPlannedRows },
     executor: defendedContract.executor,
     model: defendedContract.model,
     baseUrl: defendedContract.baseUrl,
     requestTimeoutMs: defendedContract.requestTimeoutMs,
+    backend: defendedContract.backend,
     stage: ONLINE_ADAPTIVE_STAGE,
     runKind: input.runKind ?? "dev",
     modelProfileHash: frozen.profile.modelProfileHash,
@@ -50195,11 +50297,13 @@ async function runInjectionSuiteOnlineAdaptive(rawInput, deps = DEFAULT_ONLINE_D
       variantsPerFamily: input.variantsPerFamily,
       family: input.family ?? null,
       limit: input.limit ?? null,
+      ...unslicedPlannedRows === void 0 ? {} : { unslicedPlannedRows },
       expectedRows: planned.length,
       executor: defendedContract.executor,
       model: defendedContract.model,
       baseUrl: defendedContract.baseUrl,
       requestTimeoutMs: defendedContract.requestTimeoutMs,
+      backend: defendedContract.backend,
       stage: ONLINE_ADAPTIVE_STAGE,
       runKind: input.runKind ?? "dev",
       modelProfileHash: frozen.profile.modelProfileHash,
@@ -50212,6 +50316,9 @@ async function runInjectionSuiteOnlineAdaptive(rawInput, deps = DEFAULT_ONLINE_D
       captureResponses: true,
       attackerExecutor: attacker.executor,
       attackerModel: attacker.model,
+      // Persisted so the analyzer can recompute the resume hash exactly as
+      // the runner did (PR #3081 r3); optional, so old runs are unaffected.
+      ...attacker.baseUrl ? { attackerBaseUrl: attacker.baseUrl } : {},
       attackerModelDigest: input.attackerModelDigest ?? "unverified",
       attackerPromptSha256,
       attackerIterations: input.attackerIterations
@@ -50515,37 +50622,6 @@ async function runInjectionSuiteOnlineAdaptive(rawInput, deps = DEFAULT_ONLINE_D
     resumed
   };
 }
-var INJECTION_SUITE_ONLINE_RESUME_CONTRACT = "h5-injection-suite-online-resume-v1";
-function injectionSuiteResumeContractHashForOnline(metadata) {
-  return createHash29("sha256").update(
-    JSON.stringify({
-      contract: INJECTION_SUITE_ONLINE_RESUME_CONTRACT,
-      suiteVersion: metadata.suiteVersion,
-      modelProfileId: metadata.modelProfileId,
-      seeds: metadata.seeds,
-      variantsPerFamily: metadata.variantsPerFamily,
-      family: metadata.family ?? null,
-      limit: metadata.limit,
-      executor: metadata.executor,
-      model: metadata.model,
-      baseUrl: metadata.baseUrl,
-      requestTimeoutMs: metadata.requestTimeoutMs,
-      stage: metadata.stage ?? ONLINE_ADAPTIVE_STAGE,
-      runKind: metadata.runKind ?? "dev",
-      modelProfileHash: metadata.modelProfileHash ?? "",
-      corpusManifestHash: metadata.corpusManifestHash ?? "",
-      expectedDesignHash: metadata.expectedDesignHash ?? "",
-      decisionRuleHash: metadata.decisionRuleHash ?? "",
-      gitSha: metadata.gitSha ?? "",
-      attackerExecutor: metadata.attackerExecutor ?? "",
-      attackerModel: metadata.attackerModel ?? "",
-      attackerBaseUrl: metadata.attackerBaseUrl ?? "",
-      attackerModelDigest: metadata.attackerModelDigest ?? "",
-      attackerPromptSha256: metadata.attackerPromptSha256 ?? "",
-      attackerIterations: metadata.attackerIterations ?? 0
-    })
-  ).digest("hex");
-}
 
 // src/security/injection-suite/stats.ts
 import { writeFileAtomically as writeFileAtomically7 } from "@remnic/core/maintenance/atomic-file";
@@ -50927,6 +51003,8 @@ import {
   openSync,
   readdirSync,
   readFileSync as readFileSync2,
+  realpathSync as realpathSync2,
+  statSync,
   renameSync,
   rmSync,
   writeFileSync
@@ -51223,6 +51301,51 @@ async function runDriftUtility(input, arm, seed, plannedItems) {
   }
 }
 var UTILITY_CONTRACT_FILE = "utility-contract.json";
+function datasetDigest(directory) {
+  if (!directory) return null;
+  const root = path47.resolve(directory);
+  const files = [];
+  const visited = /* @__PURE__ */ new Set();
+  const walk = (dir) => {
+    let real;
+    try {
+      real = realpathSync2(dir);
+    } catch {
+      return;
+    }
+    if (visited.has(real)) return;
+    visited.add(real);
+    for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name < b.name ? -1 : 1)) {
+      const full = path47.join(dir, entry.name);
+      if (entry.isFile()) {
+        files.push(full);
+        continue;
+      }
+      if (entry.isDirectory()) {
+        walk(full);
+        continue;
+      }
+      if (!entry.isSymbolicLink()) continue;
+      let target;
+      try {
+        target = statSync(full);
+      } catch {
+        target = void 0;
+      }
+      if (target?.isFile()) files.push(full);
+      else if (target?.isDirectory()) walk(full);
+    }
+  };
+  walk(root);
+  const digest = createHash30("sha256");
+  for (const file of files.sort()) {
+    digest.update(path47.relative(root, file).split(path47.sep).join("/"));
+    digest.update("\0");
+    digest.update(createHash30("sha256").update(readFileSync2(file)).digest("hex"));
+    digest.update("\0");
+  }
+  return `sha256:${digest.digest("hex")}:${files.length}`;
+}
 function utilityContract(input, benchmarks) {
   const executor = resolvedExecutorContract(input);
   return {
@@ -51232,12 +51355,18 @@ function utilityContract(input, benchmarks) {
     modelDigest: input.modelDigest?.trim() || "unverified",
     modelProfileId: input.modelProfileId,
     baseUrl: executor.baseUrl,
+    // The backend selects non-generic request fields, so resuming under a
+    // different override is a different condition (PR #3079 post-cap).
+    backend: executor.backend,
     benchmarks: [...benchmarks],
     seeds: [...UTILITY_SEEDS],
     limit: input.limit ?? null,
     runKind: input.runKind ?? null,
     locomoDatasetDir: input.locomoDatasetDir ?? null,
-    longmemevalDatasetDir: input.longmemevalDatasetDir ?? null
+    locomoDatasetDigest: benchmarks.includes("locomo") ? datasetDigest(input.locomoDatasetDir) : null,
+    longmemevalDatasetDir: input.longmemevalDatasetDir ?? null,
+    longmemevalDatasetDigest: benchmarks.includes("longmemeval") ? datasetDigest(input.longmemevalDatasetDir) : null,
+    driftDatasetDigest: benchmarks.includes("drift-gen") ? datasetDigest(DRIFT_ROOT) : null
   };
 }
 async function assertUtilityContract(input, benchmarks) {
@@ -51691,7 +51820,7 @@ function pickStableQualifiedName(repo, index) {
 // src/coding-graph/harness.ts
 import { performance as performance2 } from "perf_hooks";
 import { mkdtemp as mkdtemp14, rm as rm19 } from "fs/promises";
-import { statSync } from "fs";
+import { statSync as statSync2 } from "fs";
 import { tmpdir as tmpdir7 } from "os";
 import path50 from "path";
 import os10 from "os";
@@ -51866,9 +51995,9 @@ async function runCodingGraphBenchmark(config = {}) {
         throw new Error(`dead_code failed: ${deadCode.result.code}`);
       }
       await store.drain();
-      let dbBytes = statSync(dbPath).size;
+      let dbBytes = statSync2(dbPath).size;
       try {
-        dbBytes += statSync(dbPath + "-wal").size;
+        dbBytes += statSync2(dbPath + "-wal").size;
       } catch {
       }
       const kloc = Math.max(1, repo.approximateLoc / 1e3);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/bench",
-  "version": "9.69.61",
+  "version": "9.69.64",
   "description": "Retrieval latency ladder benchmarks + CI regression gates for @remnic/core",
   "type": "module",
   "main": "./dist/index.js",
@@ -41,8 +41,8 @@
     "hyparquet": "^1.25.7",
     "yaml": "^2.4.2",
     "zod": "^3.24.0",
-    "@remnic/coding-graph": "^9.69.61",
-    "@remnic/core": "^9.69.61"
+    "@remnic/coding-graph": "^9.69.64",
+    "@remnic/core": "^9.69.64"
   },
   "devDependencies": {
     "tsup": "^8.5.1",