@remnic/bench 9.69.39 → 9.69.41

package/dist/{chunk-PWFYSAUK.js → chunk-QW6VEGZN.js}

@@ -2635,6 +2635,586 @@ ${renderProvenance(result, provenance)}
 `;
 }
 
+// src/provider-config.ts
+var BUILT_IN_PROVIDERS = [
+  "openai",
+  "anthropic",
+  "ollama",
+  "litellm",
+  "local-llm",
+  "codex-cli",
+  "claude-cli"
+];
+var BENCH_REASONING_EFFORTS = ["low", "medium", "high", "xhigh"];
+function quotedList(values) {
+  const quoted = values.map((value) => `"${value}"`);
+  return quoted.length <= 1 ? quoted.join("") : `${quoted.slice(0, -1).join(", ")}, or ${quoted[quoted.length - 1]}`;
+}
+var PROVIDER_ENUM_LIST = quotedList(BUILT_IN_PROVIDERS);
+var REASONING_EFFORT_LIST = quotedList(BENCH_REASONING_EFFORTS);
+var PROVIDER_CONFIG_FIELD_MARKERS = {
+  provider: true,
+  model: true,
+  rubricVersion: true,
+  baseUrl: true,
+  apiKey: true,
+  retryOptions: true,
+  providerRequestTimeoutMs: true,
+  disableThinking: true,
+  reasoningEffort: true,
+  responderContextBudgetChars: true,
+  responderPromptBudgetChars: true,
+  temperature: true,
+  seed: true
+};
+var PROVIDER_CONFIG_VALIDATED_FIELDS = Object.keys(PROVIDER_CONFIG_FIELD_MARKERS);
+function isPositiveInteger(value) {
+  return typeof value === "number" && Number.isInteger(value) && value > 0;
+}
+function isNonNegativeFinite2(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+function isNonNegativeInteger(value) {
+  return typeof value === "number" && Number.isInteger(value) && value >= 0;
+}
+function checkOptional(container, key, accept, reason) {
+  if (!(key in container)) {
+    return null;
+  }
+  return accept(container[key]) ? null : { fieldPath: key, reason };
+}
+function firstIssue(...issues) {
+  for (const issue of issues) {
+    if (issue !== null) {
+      return issue;
+    }
+  }
+  return null;
+}
+function validateProviderConfigShape(value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    return { fieldPath: "", reason: "must be a provider config ({ provider, model })" };
+  }
+  const config = value;
+  if (!BUILT_IN_PROVIDERS.includes(config.provider)) {
+    return { fieldPath: "provider", reason: `must be one of ${PROVIDER_ENUM_LIST}` };
+  }
+  if (typeof config.model !== "string") {
+    return { fieldPath: "model", reason: "must be a string" };
+  }
+  const rootIssue = firstIssue(
+    checkOptional(config, "rubricVersion", (v) => typeof v === "string", "must be a string when present"),
+    checkOptional(config, "baseUrl", (v) => typeof v === "string", "must be a string when present"),
+    checkOptional(config, "apiKey", (v) => typeof v === "string", "must be a string when present"),
+    checkOptional(config, "providerRequestTimeoutMs", isPositiveInteger, "must be a positive integer when present"),
+    checkOptional(config, "disableThinking", (v) => typeof v === "boolean", "must be a boolean when present"),
+    checkOptional(
+      config,
+      "reasoningEffort",
+      (v) => BENCH_REASONING_EFFORTS.includes(v),
+      `must be one of ${REASONING_EFFORT_LIST} when present`
+    ),
+    checkOptional(config, "responderContextBudgetChars", isPositiveInteger, "must be a positive integer when present"),
+    checkOptional(config, "responderPromptBudgetChars", isPositiveInteger, "must be a positive integer when present"),
+    checkOptional(config, "temperature", isNonNegativeFinite2, "must be a finite non-negative number when present"),
+    checkOptional(config, "seed", isNonNegativeInteger, "must be a non-negative integer when present")
+  );
+  if (rootIssue) {
+    return rootIssue;
+  }
+  if (!("retryOptions" in config)) {
+    return null;
+  }
+  const retry = config.retryOptions;
+  if (typeof retry !== "object" || retry === null || Array.isArray(retry)) {
+    return { fieldPath: "retryOptions", reason: "must be an object when present" };
+  }
+  const retryOptions = retry;
+  const retryIssue = firstIssue(
+    checkOptional(retryOptions, "maxAttempts", isPositiveInteger, "must be a positive integer when present"),
+    checkOptional(retryOptions, "baseBackoffMs", isNonNegativeFinite2, "must be a finite non-negative number when present"),
+    checkOptional(retryOptions, "timeoutMs", isNonNegativeFinite2, "must be a finite non-negative number when present"),
+    checkOptional(retryOptions, "retryOnTimeout", (v) => typeof v === "boolean", "must be a boolean when present"),
+    checkOptional(retryOptions, "max429WaitMs", isNonNegativeFinite2, "must be a finite non-negative number when present")
+  );
+  return retryIssue ? { fieldPath: `retryOptions.${retryIssue.fieldPath}`, reason: retryIssue.reason } : null;
+}
+
+// src/legacy-artifact.ts
+var LEGACY_ARTIFACT_SHAPE_VERSION = 1;
+function isRecord2(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+function isFiniteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value);
+}
+var ArtifactRejected = class extends Error {
+  constructor(reason) {
+    super(reason);
+    this.reason = reason;
+  }
+  reason;
+};
+function reject(reason) {
+  throw new ArtifactRejected(reason);
+}
+function optionalString(where, container, key) {
+  if (!(key in container)) {
+    return void 0;
+  }
+  if (typeof container[key] !== "string") {
+    reject(`${where} must be a string when present`);
+  }
+  return container[key];
+}
+function optionalFiniteNumber(where, container, key) {
+  if (!(key in container)) {
+    return void 0;
+  }
+  if (!isFiniteNumber(container[key])) {
+    reject(`${where} must be a finite number when present`);
+  }
+  return container[key];
+}
+function optionalNonNegativeInteger(where, container, key) {
+  if (!(key in container)) {
+    return void 0;
+  }
+  const value = container[key];
+  if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
+    reject(`${where} must be a non-negative integer when present`);
+  }
+  return value;
+}
+function isDeclaredMultiSample(legacy) {
+  if (!isRecord2(legacy.meta)) {
+    return false;
+  }
+  const runCount = legacy.meta.runCount;
+  if (typeof runCount === "number" && Number.isFinite(runCount) && runCount > 1) {
+    return true;
+  }
+  return Array.isArray(legacy.meta.seeds) && legacy.meta.seeds.length > 1;
+}
+function optionalMode(where, container) {
+  if (!("mode" in container)) {
+    return void 0;
+  }
+  if (container.mode !== "quick" && container.mode !== "full") {
+    reject(`${where} must be "quick" or "full" when present`);
+  }
+  return container.mode;
+}
+function optionalTier(where, container) {
+  if (!("benchmarkTier" in container)) {
+    return void 0;
+  }
+  const value = container.benchmarkTier;
+  if (value !== "published" && value !== "remnic" && value !== "custom") {
+    reject(`${where} must be "published", "remnic", or "custom" when present`);
+  }
+  return value;
+}
+function optionalProviderConfig(where, value) {
+  if (value === void 0 || value === null) {
+    return null;
+  }
+  const issue = validateProviderConfigShape(value);
+  if (issue) {
+    reject(
+      issue.fieldPath ? `${where}.${issue.fieldPath} ${issue.reason}` : `${where} ${issue.reason} or null when present`
+    );
+  }
+  return value;
+}
+function optionalSeeds(where, value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  if (!Array.isArray(value) || !value.every((item) => typeof item === "number" && Number.isInteger(item))) {
+    reject(`${where} must be an array of integers when present`);
+  }
+  return value;
+}
+function isBenchRuntimeProfile(value) {
+  return value === "baseline" || value === "real" || value === "openclaw-chain" || value === "local-lab";
+}
+function normalizeMetricAggregate(where, raw, taskCount, declaredMultiSample) {
+  if (isRecord2(raw) && isFiniteNumber(raw.mean) && isFiniteNumber(raw.median) && isFiniteNumber(raw.stdDev) && isFiniteNumber(raw.min) && isFiniteNumber(raw.max)) {
+    return raw;
+  }
+  if (!isRecord2(raw)) {
+    reject(`${where} must be an object with a finite mean number`);
+  }
+  for (const field of ["mean", "median", "stdDev", "min", "max"]) {
+    if (field in raw && !isFiniteNumber(raw[field])) {
+      reject(`${where}.${field} must be a finite number when present`);
+    }
+  }
+  if (!isFiniteNumber(raw.mean)) {
+    reject(`${where} must be an object with a finite mean number`);
+  }
+  const rawMedian = isFiniteNumber(raw.median) ? raw.median : void 0;
+  const rawStdDev = isFiniteNumber(raw.stdDev) ? raw.stdDev : void 0;
+  const rawMin = isFiniteNumber(raw.min) ? raw.min : void 0;
+  const rawMax = isFiniteNumber(raw.max) ? raw.max : void 0;
+  if (rawMedian !== void 0 || rawStdDev !== void 0 || rawMin !== void 0 || rawMax !== void 0) {
+    reject(
+      `${where} missing required fields (median, stdDev, min, max); partial aggregates cannot mix persisted and synthesized values`
+    );
+  }
+  if (taskCount === 1 && !declaredMultiSample) {
+    return {
+      mean: raw.mean,
+      median: raw.mean,
+      stdDev: 0,
+      min: raw.mean,
+      max: raw.mean
+    };
+  }
+  if (declaredMultiSample) {
+    reject(
+      `${where} missing required multi-sample fields (median, stdDev, min, max) for declared multi-run artifact`
+    );
+  }
+  if (taskCount === 0) {
+    reject(
+      `${where} missing required fields (median, stdDev, min, max); mean-only upgrade requires exactly one recognized task`
+    );
+  }
+  reject(
+    `${where} missing required multi-sample fields (median, stdDev, min, max) for multi-task run (taskCount=${taskCount})`
+  );
+}
+function upgradeMeta(legacy, recognizedTaskCount) {
+  if (!isRecord2(legacy.meta)) {
+    reject("meta with non-empty id, benchmark, and timestamp strings is required");
+  }
+  const meta = legacy.meta;
+  for (const key of ["id", "benchmark", "timestamp"]) {
+    const value = meta[key];
+    if (typeof value !== "string" || value.trim().length === 0) {
+      reject(`meta.${key} must be a non-empty string`);
+    }
+  }
+  if (!Number.isFinite(Date.parse(meta.timestamp))) {
+    reject("meta.timestamp must be a parseable date");
+  }
+  const taskCount = recognizedTaskCount;
+  const upgraded = {
+    id: meta.id,
+    benchmark: meta.benchmark,
+    timestamp: meta.timestamp,
+    // Old UI display default for an absent tier.
+    benchmarkTier: optionalTier("meta.benchmarkTier", meta) ?? "custom",
+    // Provenance is not knowable from a legacy artifact: recognition
+    // rejects any payload with a present provenance key, so the only
+    // honest value here is the explicit "unknown" marker.
+    version: "unknown",
+    remnicVersion: "unknown",
+    gitSha: "unknown",
+    // Old UI display default for an absent mode.
+    mode: optionalMode("meta.mode", meta) ?? "quick",
+    // Old UI fell back to the task count when runCount was absent.
+    runCount: optionalNonNegativeInteger("meta.runCount", meta, "runCount") ?? taskCount,
+    seeds: optionalSeeds("meta.seeds", meta.seeds) ?? []
+  };
+  const metaExtras = upgraded;
+  for (const key of [
+    "runId",
+    "gitDirty",
+    "gitDirtyEntryCount",
+    "splitType",
+    "qrelsSealedHash",
+    "judgePromptHash",
+    "datasetHash",
+    "canaryScore",
+    "canaryFloor",
+    "status",
+    "failureReason"
+  ]) {
+    if (!(key in meta)) {
+      continue;
+    }
+    if (key === "canaryFloor") {
+      const floorVal = optionalFiniteNumber("meta.canaryFloor", meta, "canaryFloor");
+      if (floorVal !== void 0) {
+        if (floorVal < 0) {
+          reject("meta.canaryFloor must be a non-negative number when present");
+        }
+        metaExtras[key] = floorVal;
+      }
+      continue;
+    }
+    if (key === "canaryScore" || key === "gitDirtyEntryCount") {
+      const numeric = optionalFiniteNumber(`meta.${key}`, meta, key);
+      if (numeric !== void 0) {
+        metaExtras[key] = numeric;
+      }
+      continue;
+    }
+    if (key === "runId" || key === "qrelsSealedHash" || key === "judgePromptHash" || key === "datasetHash" || key === "failureReason") {
+      const text = optionalString(`meta.${key}`, meta, key);
+      if (text !== void 0) {
+        metaExtras[key] = text;
+      }
+      continue;
+    }
+    if (key === "gitDirty") {
+      if (typeof meta.gitDirty !== "boolean") {
+        reject("meta.gitDirty must be a boolean when present");
+      }
+      metaExtras.gitDirty = meta.gitDirty;
+      continue;
+    }
+    if (key === "status") {
+      if (meta.status !== "complete" && meta.status !== "partial") {
+        reject('meta.status must be "complete" or "partial" when present');
+      }
+      metaExtras.status = meta.status;
+      continue;
+    }
+    if (meta.splitType !== "public" && meta.splitType !== "holdout") {
+      reject('meta.splitType must be "public" or "holdout" when present');
+    }
+    metaExtras.splitType = meta.splitType;
+  }
+  return upgraded;
+}
+function upgradeConfig(legacy) {
+  if (!("config" in legacy)) {
+    return {
+      systemProvider: null,
+      judgeProvider: null,
+      adapterMode: "unknown",
+      remnicConfig: {}
+    };
+  }
+  if (!isRecord2(legacy.config)) {
+    reject("config must be an object when present");
+  }
+  const config = legacy.config;
+  const upgraded = {
+    systemProvider: optionalProviderConfig("config.systemProvider", config.systemProvider),
+    judgeProvider: optionalProviderConfig("config.judgeProvider", config.judgeProvider),
+    // Old UI display default for an absent adapter mode.
+    adapterMode: optionalString("config.adapterMode", config, "adapterMode") ?? "unknown",
+    remnicConfig: {}
+  };
+  if ("remnicConfig" in config) {
+    if (!isRecord2(config.remnicConfig)) {
+      reject("config.remnicConfig must be an object when present");
+    }
+    upgraded.remnicConfig = config.remnicConfig;
+  }
+  if ("internalProvider" in config && config.internalProvider !== void 0) {
+    upgraded.internalProvider = optionalProviderConfig("config.internalProvider", config.internalProvider);
+  }
+  if ("runtimeProfile" in config && config.runtimeProfile !== void 0) {
+    const profile = config.runtimeProfile;
+    if (profile !== null && !isBenchRuntimeProfile(profile)) {
+      reject(
+        'config.runtimeProfile must be "baseline", "real", "openclaw-chain", "local-lab", or null when present'
+      );
+    }
+    upgraded.runtimeProfile = profile;
+  }
+  if ("benchmarkOptions" in config && config.benchmarkOptions !== void 0) {
+    if (!isRecord2(config.benchmarkOptions)) {
+      reject("config.benchmarkOptions must be an object when present");
+    }
+    upgraded.benchmarkOptions = config.benchmarkOptions;
+  }
+  return upgraded;
+}
+function upgradeCost(legacy) {
+  if (!("cost" in legacy)) {
+    return {
+      totalTokens: 0,
+      inputTokens: 0,
+      outputTokens: 0,
+      estimatedCostUsd: 0,
+      totalLatencyMs: 0,
+      meanQueryLatencyMs: 0
+    };
+  }
+  if (!isRecord2(legacy.cost)) {
+    reject("cost must be an object when present");
+  }
+  const cost = legacy.cost;
+  const upgraded = {
+    totalTokens: optionalFiniteNumber("cost.totalTokens", cost, "totalTokens") ?? 0,
+    inputTokens: optionalFiniteNumber("cost.inputTokens", cost, "inputTokens") ?? 0,
+    outputTokens: optionalFiniteNumber("cost.outputTokens", cost, "outputTokens") ?? 0,
+    estimatedCostUsd: optionalFiniteNumber("cost.estimatedCostUsd", cost, "estimatedCostUsd") ?? 0,
+    totalLatencyMs: optionalFiniteNumber("cost.totalLatencyMs", cost, "totalLatencyMs") ?? 0,
+    meanQueryLatencyMs: optionalFiniteNumber("cost.meanQueryLatencyMs", cost, "meanQueryLatencyMs") ?? 0
+  };
+  if ("judgeModelCalls" in cost) {
+    upgraded.judgeModelCalls = optionalFiniteNumber("cost.judgeModelCalls", cost, "judgeModelCalls");
+  }
+  return upgraded;
+}
+function upgradeTask(where, task) {
+  if (!isRecord2(task) || typeof task.taskId !== "string" || task.taskId.trim().length === 0) {
+    return null;
+  }
+  if ("scores" in task && (!isRecord2(task.scores) || !Object.values(task.scores).every(isFiniteNumber))) {
+    reject(`${where}.scores must map metric names to finite numbers when present`);
+  }
+  if ("tokens" in task && !isRecord2(task.tokens)) {
+    reject(`${where}.tokens must be an object when present`);
+  }
+  const tokensSource = isRecord2(task.tokens) ? task.tokens : {};
+  const upgraded = {
+    taskId: task.taskId,
+    // Old UI display defaults for absent task text fields.
+    question: optionalString(`${where}.question`, task, "question") ?? "",
+    expected: optionalString(`${where}.expected`, task, "expected") ?? "",
+    actual: optionalString(`${where}.actual`, task, "actual") ?? "",
+    scores: isRecord2(task.scores) ? task.scores : {},
+    latencyMs: optionalFiniteNumber(`${where}.latencyMs`, task, "latencyMs") ?? 0,
+    tokens: {
+      input: optionalFiniteNumber(`${where}.tokens.input`, tokensSource, "input") ?? 0,
+      output: optionalFiniteNumber(`${where}.tokens.output`, tokensSource, "output") ?? 0
+    }
+  };
+  const taskExtras = upgraded;
+  for (const key of ["goldMemories", "attributionWitness", "details"]) {
+    if (key in task) {
+      taskExtras[key] = task[key];
+    }
+  }
+  return upgraded;
+}
+function upgradeResults(legacy) {
+  const upgraded = { tasks: [], aggregates: {} };
+  if (!("results" in legacy)) {
+    return upgraded;
+  }
+  if (!isRecord2(legacy.results)) {
+    reject("results must be an object when present");
+  }
+  const results = legacy.results;
+  const declaredMultiSample = isDeclaredMultiSample(legacy);
+  if ("tasks" in results) {
+    if (!Array.isArray(results.tasks)) {
+      reject("results.tasks must be an array when present");
+    }
+    upgraded.tasks = results.tasks.map((task, index) => upgradeTask(`results.tasks[${index}]`, task)).filter((task) => task !== null);
+  }
+  if ("aggregates" in results) {
+    if (!isRecord2(results.aggregates)) {
+      reject("results.aggregates must be an object when present");
+    }
+    const normalizedAggregates = {};
+    for (const [metricName, rawValue] of Object.entries(results.aggregates)) {
+      normalizedAggregates[metricName] = normalizeMetricAggregate(
+        `results.aggregates.${metricName}`,
+        rawValue,
+        upgraded.tasks.length,
+        declaredMultiSample
+      );
+    }
+    upgraded.aggregates = normalizedAggregates;
+  }
+  const resultsExtras = upgraded;
+  if ("categoryAggregates" in results && results.categoryAggregates !== void 0) {
+    if (!isRecord2(results.categoryAggregates)) {
+      reject("results.categoryAggregates must be an object when present");
+    }
+    const normalizedCategoryAggregates = {};
+    for (const [catName, catAggs] of Object.entries(results.categoryAggregates)) {
+      if (!isRecord2(catAggs)) {
+        reject(`results.categoryAggregates.${catName} must be an object`);
+      }
+      const catNormalized = {};
+      for (const [metricName, rawVal] of Object.entries(catAggs)) {
+        catNormalized[metricName] = normalizeMetricAggregate(
+          `results.categoryAggregates.${catName}.${metricName}`,
+          rawVal,
+          upgraded.tasks.length,
+          declaredMultiSample
+        );
+      }
+      normalizedCategoryAggregates[catName] = catNormalized;
+    }
+    resultsExtras.categoryAggregates = normalizedCategoryAggregates;
+  }
+  if ("statistics" in results) {
+    resultsExtras.statistics = results.statistics;
+  }
+  const hasCategoryAggregate = isRecord2(resultsExtras.categoryAggregates) && Object.values(resultsExtras.categoryAggregates).some(
+    (category) => isRecord2(category) && Object.keys(category).length > 0
+  );
+  if (upgraded.tasks.length === 0 && Object.keys(upgraded.aggregates).length === 0 && !hasCategoryAggregate) {
+    reject("results must contain at least one recognized task or aggregate");
+  }
+  return upgraded;
+}
+function upgradeEnvironment(legacy) {
+  if (!("environment" in legacy)) {
+    return { os: "unknown", nodeVersion: "unknown" };
+  }
+  if (!isRecord2(legacy.environment)) {
+    reject("environment must be an object when present");
+  }
+  const environment = legacy.environment;
+  const upgraded = {
+    os: optionalString("environment.os", environment, "os") ?? "unknown",
+    nodeVersion: optionalString("environment.nodeVersion", environment, "nodeVersion") ?? "unknown"
+  };
+  if ("hardware" in environment) {
+    upgraded.hardware = environment.hardware;
+  }
+  return upgraded;
+}
+function recognizeLegacyBenchmarkArtifact(value) {
+  if (!isRecord2(value)) {
+    return { ok: false, reason: "artifact is not a JSON object" };
+  }
+  if (isRecord2(value.meta)) {
+    const meta = value.meta;
+    const modernProvenance = "version" in meta || "remnicVersion" in meta || "gitSha" in meta;
+    const modernTier = meta.benchmarkTier === "published" || meta.benchmarkTier === "remnic";
+    const modernIntegrity = INTEGRITY_META_FIELDS.some((key) => key in meta);
+    if (modernProvenance || modernTier || modernIntegrity) {
+      return {
+        ok: false,
+        reason: !("results" in value) ? "modern provenance/integrity markers present; missing results is not a legacy artifact" : "modern provenance/integrity markers present; not a legacy artifact"
+      };
+    }
+  }
+  try {
+    return {
+      ok: true,
+      shapeVersion: LEGACY_ARTIFACT_SHAPE_VERSION,
+      result: (() => {
+        const results = upgradeResults(value);
+        const upgraded = {
+          meta: upgradeMeta(value, results.tasks.length),
+          config: upgradeConfig(value),
+          cost: upgradeCost(value),
+          results,
+          environment: upgradeEnvironment(value)
+        };
+        if (!("results" in value)) {
+          reject("results must contain at least one recognized task or aggregate");
+        }
+        return upgraded;
+      })()
+    };
+  } catch (error) {
+    if (error instanceof ArtifactRejected) {
+      return { ok: false, reason: error.reason };
+    }
+    return {
+      ok: false,
+      reason: error instanceof Error ? error.message : "legacy artifact upgrade failed"
+    };
+  }
+}
+
 // src/results-store.ts
 var BASELINE_NAME_PATTERN = /^[A-Za-z0-9_-]+$/;
 var REPRO_MANIFEST_FILENAME = "MANIFEST.json";
@@ -2667,7 +3247,7 @@ function isBenchmarkMode(value) {
 function isObjectRecord(value) {
   return !!value && typeof value === "object" && !Array.isArray(value);
 }
-function isFiniteNumber(value) {
+function isFiniteNumber2(value) {
   return typeof value === "number" && Number.isFinite(value);
 }
 async function loadBenchmarkReportCardProvenance(outputDir, resultId) {
@@ -2693,12 +3273,12 @@ function isProviderConfigLike(value) {
   if (value === null) {
     return true;
   }
-  return isObjectRecord(value) && typeof value.provider === "string" && typeof value.model === "string";
+  return validateProviderConfigShape(value) === null;
 }
 function isNonEmptyString(value) {
   return typeof value === "string" && value.trim().length > 0;
 }
-function isNonNegativeInteger(value) {
+function isNonNegativeInteger2(value) {
   return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
 }
 function isUniqueMemoryIdList(value) {
@@ -2716,7 +3296,7 @@ function isTaskAttributionWitnessLike(value, goldMemories) {
     return false;
   }
   const runtime = value.runtime;
-  if (!isObjectRecord(runtime) || !isNonEmptyString(runtime.qmdCollection) || !isNonEmptyString(runtime.qmdIndex) || !isNonNegativeInteger(runtime.qmdMaxResults) || typeof runtime.attributionThreshold !== "number" || !Number.isFinite(runtime.attributionThreshold) || runtime.attributionThreshold < 0 || runtime.attributionThreshold > 1) {
+  if (!isObjectRecord(runtime) || !isNonEmptyString(runtime.qmdCollection) || !isNonEmptyString(runtime.qmdIndex) || !isNonNegativeInteger2(runtime.qmdMaxResults) || typeof runtime.attributionThreshold !== "number" || !Number.isFinite(runtime.attributionThreshold) || runtime.attributionThreshold < 0 || runtime.attributionThreshold > 1) {
     return false;
   }
   if (!Array.isArray(value.golds) || goldMemories !== void 0 && value.golds.length !== goldMemories.length) {
@@ -2733,7 +3313,7 @@ function isTaskAttributionWitnessLike(value, goldMemories) {
   }
   const sessionIds = /* @__PURE__ */ new Set();
   for (const retrieval of value.retrievals) {
-    if (!isObjectRecord(retrieval) || !isNonEmptyString(retrieval.sessionId) || sessionIds.has(retrieval.sessionId) || !(retrieval.appliedCap === null || isNonNegativeInteger(retrieval.appliedCap)) || !isUniqueMemoryIdList(retrieval.atCapMemoryIds) || !isUniqueMemoryIdList(retrieval.headroomMemoryIds)) {
+    if (!isObjectRecord(retrieval) || !isNonEmptyString(retrieval.sessionId) || sessionIds.has(retrieval.sessionId) || !(retrieval.appliedCap === null || isNonNegativeInteger2(retrieval.appliedCap)) || !isUniqueMemoryIdList(retrieval.atCapMemoryIds) || !isUniqueMemoryIdList(retrieval.headroomMemoryIds)) {
       return false;
     }
     sessionIds.add(retrieval.sessionId);
@@ -2752,6 +3332,9 @@ function isTaskAttributionWitnessLike(value, goldMemories) {
   }
   return true;
 }
+function isMetricAggregate(value) {
+  return isObjectRecord(value) && isFiniteNumber2(value.mean) && isFiniteNumber2(value.median) && isFiniteNumber2(value.stdDev) && isFiniteNumber2(value.min) && isFiniteNumber2(value.max);
+}
 function isBenchmarkResult(value) {
   if (!isObjectRecord(value)) {
     return false;
@@ -2760,7 +3343,7 @@ function isBenchmarkResult(value) {
   if (!isObjectRecord(meta)) {
     return false;
   }
-  const hasValidMeta = typeof meta.id === "string" && typeof meta.benchmark === "string" && (meta.benchmarkTier === "published" || meta.benchmarkTier === "remnic" || meta.benchmarkTier === "custom") && typeof meta.version === "string" && typeof meta.remnicVersion === "string" && typeof meta.gitSha === "string" && typeof meta.timestamp === "string" && isBenchmarkMode(meta.mode) && isFiniteNumber(meta.runCount) && Array.isArray(meta.seeds) && meta.seeds.every(isFiniteNumber);
+  const hasValidMeta = typeof meta.id === "string" && typeof meta.benchmark === "string" && (meta.benchmarkTier === "published" || meta.benchmarkTier === "remnic" || meta.benchmarkTier === "custom") && typeof meta.version === "string" && typeof meta.remnicVersion === "string" && typeof meta.gitSha === "string" && typeof meta.timestamp === "string" && isBenchmarkMode(meta.mode) && isFiniteNumber2(meta.runCount) && Array.isArray(meta.seeds) && meta.seeds.every(isFiniteNumber2) && (meta.canaryFloor === void 0 || isFiniteNumber2(meta.canaryFloor) && meta.canaryFloor >= 0) && (meta.canaryScore === void 0 || isFiniteNumber2(meta.canaryScore)) && (meta.failureReason === void 0 || typeof meta.failureReason === "string") && (meta.runId === void 0 || typeof meta.runId === "string") && (meta.gitDirty === void 0 || typeof meta.gitDirty === "boolean") && (meta.gitDirtyEntryCount === void 0 || isFiniteNumber2(meta.gitDirtyEntryCount)) && (meta.splitType === void 0 || meta.splitType === "public" || meta.splitType === "holdout") && (meta.qrelsSealedHash === void 0 || typeof meta.qrelsSealedHash === "string") && (meta.judgePromptHash === void 0 || typeof meta.judgePromptHash === "string") && (meta.datasetHash === void 0 || typeof meta.datasetHash === "string") && (meta.status === void 0 || meta.status === "complete" || meta.status === "partial");
   if (!hasValidMeta) {
     return false;
   }
@@ -2769,13 +3352,23 @@ function isBenchmarkResult(value) {
     return false;
   }
   const cost = value.cost;
-  if (!isObjectRecord(cost) || !isFiniteNumber(cost.totalTokens) || !isFiniteNumber(cost.inputTokens) || !isFiniteNumber(cost.outputTokens) || !isFiniteNumber(cost.estimatedCostUsd) || !isFiniteNumber(cost.totalLatencyMs) || !isFiniteNumber(cost.meanQueryLatencyMs)) {
+  if (!isObjectRecord(cost) || !isFiniteNumber2(cost.totalTokens) || !isFiniteNumber2(cost.inputTokens) || !isFiniteNumber2(cost.outputTokens) || !isFiniteNumber2(cost.estimatedCostUsd) || !isFiniteNumber2(cost.totalLatencyMs) || !isFiniteNumber2(cost.meanQueryLatencyMs)) {
     return false;
   }
   const results = value.results;
-  if (!isObjectRecord(results) || !Array.isArray(results.tasks) || !results.tasks.every(isTaskResultLike) || !isObjectRecord(results.aggregates)) {
+  if (!isObjectRecord(results) || !Array.isArray(results.tasks) || !results.tasks.every(isTaskResultLike) || !isObjectRecord(results.aggregates) || !Object.values(results.aggregates).every(isMetricAggregate)) {
     return false;
   }
+  if (results.categoryAggregates !== void 0) {
+    if (!isObjectRecord(results.categoryAggregates)) {
+      return false;
+    }
+    for (const catAgg of Object.values(results.categoryAggregates)) {
+      if (!isObjectRecord(catAgg) || !Object.values(catAgg).every(isMetricAggregate)) {
+        return false;
+      }
+    }
+  }
   const environment = value.environment;
   return isObjectRecord(environment) && typeof environment.os === "string" && typeof environment.nodeVersion === "string" && (environment.hardware === void 0 || typeof environment.hardware === "string");
 }
@@ -2791,7 +3384,7 @@ function isTaskResultLike(value) {
   if (value.attributionWitness !== void 0 && !isTaskAttributionWitnessLike(value.attributionWitness, goldMemories)) {
     return false;
   }
-  return typeof value.taskId === "string" && typeof value.question === "string" && typeof value.expected === "string" && typeof value.actual === "string" && isObjectRecord(value.scores) && Object.values(value.scores).every(isFiniteNumber) && isFiniteNumber(value.latencyMs) && isObjectRecord(tokens) && isFiniteNumber(tokens.input) && isFiniteNumber(tokens.output);
+  return typeof value.taskId === "string" && typeof value.question === "string" && typeof value.expected === "string" && typeof value.actual === "string" && isObjectRecord(value.scores) && Object.values(value.scores).every(isFiniteNumber2) && isFiniteNumber2(value.latencyMs) && isObjectRecord(tokens) && isFiniteNumber2(tokens.input) && isFiniteNumber2(tokens.output);
 }
 function isStoredBenchmarkBaseline(value) {
   if (!value || typeof value !== "object") {
@@ -2847,10 +3440,16 @@ function toBaselineSummary(baseline, filePath) {
 async function loadBenchmarkResult(filePath) {
   const content = await readFile3(filePath, "utf8");
   const parsed = JSON.parse(content);
-  if (!isBenchmarkResult(parsed)) {
-    throw new Error(`Invalid benchmark result file: ${filePath}`);
+  if (isBenchmarkResult(parsed)) {
+    return parsed;
   }
-  return parsed;
+  const legacy = recognizeLegacyBenchmarkArtifact(parsed);
+  if (legacy.ok && isBenchmarkResult(legacy.result)) {
+    return legacy.result;
+  }
+  throw new Error(
+    `Invalid benchmark result file: ${filePath}${legacy.ok ? " (legacy artifact failed canonical re-validation)" : ` (${legacy.reason})`}`
+  );
 }
 async function listBenchmarkResults(outputDir) {
   if (!fs.existsSync(outputDir)) {
@@ -3372,10 +3971,10 @@ function sha256Buffer(value) {
 }
 async function sha256File(filePath) {
   const hash = createHash3("sha256");
-  await new Promise((resolve2, reject) => {
+  await new Promise((resolve2, reject2) => {
     const stream = createReadStream(filePath);
     stream.on("data", (chunk) => hash.update(chunk));
-    stream.on("error", reject);
+    stream.on("error", reject2);
     stream.on("end", resolve2);
   });
   return hash.digest("hex");
@@ -13257,8 +13856,8 @@ function finalizeResult(state, status, invalidReason) {
 }
 function raceAbort(operation, signal) {
   if (signal.aborted) return Promise.reject(new DOMException("aborted", "AbortError"));
-  return new Promise((resolve2, reject) => {
-    const onAbort = () => reject(new DOMException("aborted", "AbortError"));
+  return new Promise((resolve2, reject2) => {
+    const onAbort = () => reject2(new DOMException("aborted", "AbortError"));
     signal.addEventListener("abort", onAbort, { once: true });
     operation.then(
       (value) => {
@@ -13267,7 +13866,7 @@ function raceAbort(operation, signal) {
       },
       (error) => {
         signal.removeEventListener("abort", onAbort);
-        reject(error);
+        reject2(error);
       }
     );
   });
@@ -14049,8 +14648,8 @@ function finalizeResult2(state, status, invalidReason, evidence) {
 }
 async function raceSignal(operation, signal) {
   if (signal.aborted) throw new DOMException("aborted", "AbortError");
-  const { promise: aborted, reject } = Promise.withResolvers();
-  const onAbort = () => reject(new DOMException("aborted", "AbortError"));
+  const { promise: aborted, reject: reject2 } = Promise.withResolvers();
+  const onAbort = () => reject2(new DOMException("aborted", "AbortError"));
   signal.addEventListener("abort", onAbort, { once: true });
   try {
     return await Promise.race([operation, aborted]);

package/dist/index.d.ts

@@ -7663,6 +7663,7 @@ declare const H6BenchmarkDatasetSchema: z.ZodObject<{
     }>;
 }, "strip", z.ZodTypeAny, {
     version: 1;
+    seed: number;
     tasks: {
         title: string;
         split: "dev" | "main" | "pilot";
@@ -7754,7 +7755,6 @@ declare const H6BenchmarkDatasetSchema: z.ZodObject<{
             noTrapRevisionSha: string;
         }[];
     }[];
-    seed: number;
     createdAt: string;
     inventoryHash: string;
     supportArtifactHashes: {
@@ -7780,6 +7780,7 @@ declare const H6BenchmarkDatasetSchema: z.ZodObject<{
     };
 }, {
     version: 1;
+    seed: number;
     tasks: {
         title: string;
         split: "dev" | "main" | "pilot";
@@ -7871,7 +7872,6 @@ declare const H6BenchmarkDatasetSchema: z.ZodObject<{
             noTrapRevisionSha: string;
         }[];
     }[];
-    seed: number;
     createdAt: string;
     inventoryHash: string;
     supportArtifactHashes: {

package/dist/index.js

@@ -187,7 +187,7 @@ import {
   writeLeaderboardArtifactsForResult,
   writeRepeatedFailureRunMetadata,
   writeRepeatedFailureStatistics
-} from "./chunk-PWFYSAUK.js";
+} from "./chunk-QW6VEGZN.js";
 
 // src/build-week-evidence-receipt.ts
 import { createHash as createHash2 } from "crypto";
@@ -46755,13 +46755,83 @@ function buildRecallPrompt(identity, variant) {
 function trimSlash(url) {
   return url.replace(/\/+$/, "");
 }
-async function postJson(url, body, timeoutMs) {
+function parseCompatUrl(baseUrl) {
+  try {
+    const parsed = new URL(baseUrl);
+    let hostname2 = parsed.hostname.trim().toLowerCase();
+    while (hostname2.endsWith(".")) hostname2 = hostname2.slice(0, -1);
+    if (hostname2.length === 0) return void 0;
+    return { protocol: parsed.protocol.toLowerCase(), hostname: hostname2 };
+  } catch {
+    return void 0;
+  }
+}
+var OPENAI_API_HOSTS = Object.freeze(["api.openai.com"]);
+var NVIDIA_API_HOSTS = Object.freeze(["integrate.api.nvidia.com"]);
+function isExactAllowlistedHost(hostname2, allowlist) {
+  return allowlist.includes(hostname2);
+}
+function isHttps(protocol) {
+  return protocol === "https:";
+}
+function isLoopbackHttpHost(hostname2) {
+  return hostname2 === "127.0.0.1" || hostname2 === "localhost";
+}
+function requireHttps(protocol, message) {
+  if (!isHttps(protocol)) {
+    throw new InjectionSuiteHostFault(message);
+  }
+}
+function nonEmptyEnv(name) {
+  const raw = process.env[name];
+  if (typeof raw !== "string") return void 0;
+  const trimmed = raw.trim();
+  return trimmed.length > 0 ? trimmed : void 0;
+}
+function requireEnvToken(envName, message) {
+  const token = nonEmptyEnv(envName);
+  if (token === void 0) {
+    throw new InjectionSuiteHostFault(message);
+  }
+  return token;
+}
+function resolveOpenAiCompatToken(baseUrl) {
+  const parsed = parseCompatUrl(baseUrl);
+  if (parsed === void 0) {
+    throw new InjectionSuiteHostFault("openai-compat requires a valid http(s) base URL");
+  }
+  const { protocol, hostname: hostname2 } = parsed;
+  if (isExactAllowlistedHost(hostname2, NVIDIA_API_HOSTS)) {
+    requireHttps(protocol, "openai-compat NVIDIA host requires https");
+    return requireEnvToken(
+      "NVIDIA_API_KEY",
+      "openai-compat NVIDIA host requires NVIDIA_API_KEY"
+    );
+  }
+  if (isExactAllowlistedHost(hostname2, OPENAI_API_HOSTS)) {
+    requireHttps(protocol, "openai-compat OpenAI host requires https");
+    return requireEnvToken(
+      "OPENAI_API_KEY",
+      "openai-compat OpenAI host requires OPENAI_API_KEY"
+    );
+  }
+  if (!isHttps(protocol) && !isLoopbackHttpHost(hostname2)) {
+    throw new InjectionSuiteHostFault(
+      "openai-compat custom host requires https (loopback HTTP is allowed only for 127.0.0.1 or localhost)"
+    );
+  }
+  return requireEnvToken(
+    "REMNIC_OPENAI_COMPAT_API_KEY",
+    "openai-compat unknown host requires REMNIC_OPENAI_COMPAT_API_KEY (or a known host: api.openai.com / integrate.api.nvidia.com); do not reuse OPENAI_API_KEY or NVIDIA_API_KEY"
+  );
+}
+async function postJson(url, body, timeoutMs, extraHeaders) {
   const controller = new AbortController();
   const timer = setTimeout(() => controller.abort(), timeoutMs);
   try {
     const response = await fetch(url, {
       method: "POST",
-      headers: { "content-type": "application/json" },
+      headers: { "content-type": "application/json", ...extraHeaders },
       body: JSON.stringify(body),
       signal: controller.signal
     });
@@ -46782,11 +46852,17 @@ async function completeChat(options, prompt) {
   const model = options.model ?? DEFAULT_OLLAMA_MODEL;
   if (options.kind === "openai-compat") {
     const base2 = trimSlash(options.baseUrl ?? DEFAULT_OPENAI_COMPAT_BASE_URL);
-    const json2 = await postJson(`${base2}/chat/completions`, {
-      model,
-      messages: [{ role: "user", content: prompt }],
-      temperature: 0
-    }, timeoutMs);
+    const token = resolveOpenAiCompatToken(base2);
+    const json2 = await postJson(
+      `${base2}/chat/completions`,
+      {
+        model,
+        messages: [{ role: "user", content: prompt }],
+        temperature: 0
+      },
+      timeoutMs,
+      { Authorization: `Bearer ${token}` }
+    );
     const text2 = json2.choices?.[0]?.message?.content;
     if (typeof text2 !== "string") throw new InjectionSuiteHostFault("openai-compat response missing content");
     return text2;
@@ -48908,7 +48984,7 @@ async function writeRepeatedFailurePaperArtifacts(options) {
 }
 async function runRepeatedFailurePaperReportCliCommand(options) {
   try {
-    const { replayRepeatedFailureStatistics: replayRepeatedFailureStatistics2 } = await import("./repeated-failure-suite-runner-ONGURPYC.js");
+    const { replayRepeatedFailureStatistics: replayRepeatedFailureStatistics2 } = await import("./repeated-failure-suite-runner-GEBPBTNL.js");
     const replay = await replayRepeatedFailureStatistics2(options);
     if (replay.exitCode !== 0) return replay;
     const result = await writeRepeatedFailurePaperArtifacts(options);

package/dist/{repeated-failure-suite-runner-ONGURPYC.js → repeated-failure-suite-runner-GEBPBTNL.js}

@@ -11,7 +11,7 @@ import {
   runRepeatedFailureCliCommand,
   runRepeatedFailureSuite,
   verifyPreregistrationBinding
-} from "./chunk-PWFYSAUK.js";
+} from "./chunk-QW6VEGZN.js";
 export {
   baseEvidence,
   buildDesign,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/bench",
-  "version": "9.69.39",
+  "version": "9.69.41",
   "description": "Retrieval latency ladder benchmarks + CI regression gates for @remnic/core",
   "type": "module",
   "main": "./dist/index.js",
@@ -41,8 +41,8 @@
     "hyparquet": "^1.25.7",
     "yaml": "^2.4.2",
     "zod": "^3.24.0",
-    "@remnic/coding-graph": "^9.69.39",
-    "@remnic/core": "^9.69.39"
+    "@remnic/coding-graph": "^9.69.41",
+    "@remnic/core": "^9.69.41"
   },
   "devDependencies": {
     "tsup": "^8.5.1",