@remnic/bench 9.3.681 → 9.3.683

package/dist/index.js

@@ -7663,8 +7663,9 @@ var OpenAiCompatibleProvider = class {
             ...opts.systemPrompt ? [{ role: "system", content: opts.systemPrompt }] : [],
             { role: "user", content: prompt }
           ],
-          temperature: opts.temperature,
+          temperature: opts.temperature ?? this.config.temperature,
           max_tokens: opts.maxTokens,
+          ...this.config.seed !== void 0 ? { seed: this.config.seed } : {},
           ...this.config.disableThinking && isThinkingCompatibleBackend(this.config.baseUrl) ? { chat_template_kwargs: { enable_thinking: false } } : {}
         })
       },
@@ -7857,8 +7858,9 @@ var LocalLlmProvider = class {
               ...opts.systemPrompt ? [{ role: "system", content: opts.systemPrompt }] : [],
               { role: "user", content: prompt }
             ],
-            temperature: opts.temperature,
+            temperature: opts.temperature ?? this.config.temperature,
             max_tokens: opts.maxTokens,
+            ...this.config.seed !== void 0 ? { seed: this.config.seed } : {},
             ...this.config.disableThinking && isThinkingCompatibleBackend(this.normalizedBaseUrl()) ? { chat_template_kwargs: { enable_thinking: false } } : {}
           })
         },
@@ -8012,8 +8014,9 @@ var OllamaProvider = class {
           system: opts.systemPrompt,
           stream: false,
           options: {
-            temperature: opts.temperature,
-            num_predict: opts.maxTokens
+            temperature: opts.temperature ?? this.config.temperature,
+            num_predict: opts.maxTokens,
+            ...this.config.seed !== void 0 ? { seed: this.config.seed } : {}
           }
         })
       },
@@ -9656,7 +9659,7 @@ function clampNormalizedScore(value) {
 
 // src/runtime-profiles.ts
 import path10 from "path";
-import { readFile as readFile8 } from "fs/promises";
+import { readFile as readFile9 } from "fs/promises";
 import {
   resolvePluginEntry,
   setCodexCliFallbackRunnerForProcess
@@ -10003,6 +10006,469 @@ function buildPromptSpecificRequirements(prompt) {
   return requirements;
 }
 
+// src/local-lab/manifest.ts
+import { readFile as readFile8 } from "fs/promises";
+var LOCAL_LAB_PROVIDER_KINDS = [
+  "openai-compatible",
+  "ollama"
+];
+function parseLocalLabManifest(raw) {
+  if (!isPlainObject(raw)) {
+    throw new Error(
+      "local-lab manifest must be a JSON object (rule 18: parsed JSON must be object-not-null)"
+    );
+  }
+  if (raw.profile !== "local-lab") {
+    throw new Error(
+      `local-lab manifest requires profile === "local-lab"; received ${describeValue(raw.profile)}`
+    );
+  }
+  if (raw.phases !== "sequential") {
+    throw new Error(
+      `local-lab manifest phases must be "sequential" in PR2; received ${describeValue(raw.phases)}`
+    );
+  }
+  const responder = parseRole(raw.responder, "responder");
+  const judge = parseRole(raw.judge, "judge");
+  const embedding = raw.embedding === void 0 ? void 0 : parseRole(raw.embedding, "embedding");
+  const notes = raw.notes === void 0 ? void 0 : parseNotes(raw.notes, "notes");
+  return {
+    profile: "local-lab",
+    responder,
+    judge,
+    ...embedding ? { embedding } : {},
+    phases: "sequential",
+    ...notes ? { notes } : {}
+  };
+}
+async function loadLocalLabManifest(filePath) {
+  let text;
+  try {
+    text = await readFile8(filePath, "utf8");
+  } catch (error) {
+    const code = error?.code ?? "EUNKNOWN";
+    throw new Error(`local-lab manifest at ${filePath} could not be read (${code})`);
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(text);
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new Error(`local-lab manifest at ${filePath} contains invalid JSON: ${detail}`);
+  }
+  return parseLocalLabManifest(parsed);
+}
+function parseRole(value, label) {
+  if (!isPlainObject(value)) {
+    throw new Error(
+      `local-lab manifest ${label} must be an object; received ${describeValue(value)}`
+    );
+  }
+  const provider = parseProviderKind(value.provider, label);
+  const baseUrl = parseNonEmptyString(value.baseUrl, label, "baseUrl");
+  const model = parseNonEmptyString(value.model, label, "model");
+  const ctx = parsePositiveInteger(value.ctx, label, "ctx");
+  const seed = parseInteger(value.seed, label, "seed");
+  if (value.temperature !== 0) {
+    throw new Error(
+      `local-lab manifest ${label}.temperature must be the number 0; received ${describeValue(value.temperature)}`
+    );
+  }
+  const quantization = value.quantization === void 0 ? void 0 : parseNonEmptyString(value.quantization, label, "quantization");
+  return {
+    provider,
+    baseUrl,
+    model,
+    ctx,
+    temperature: 0,
+    seed,
+    ...quantization ? { quantization } : {}
+  };
+}
+function parseProviderKind(value, label) {
+  for (const kind of LOCAL_LAB_PROVIDER_KINDS) {
+    if (value === kind) {
+      return kind;
+    }
+  }
+  throw new Error(
+    `local-lab manifest ${label}.provider must be one of [${LOCAL_LAB_PROVIDER_KINDS.join(
+      ", "
+    )}]; received ${describeValue(value)}`
+  );
+}
+function parseNonEmptyString(value, label, field) {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new Error(
+      `local-lab manifest ${label}.${field} must be a non-empty string; received ${describeValue(value)}`
+    );
+  }
+  return value.trim();
+}
+function parsePositiveInteger(value, label, field) {
+  if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
+    throw new Error(
+      `local-lab manifest ${label}.${field} must be a positive integer; received ${describeValue(value)}`
+    );
+  }
+  return value;
+}
+function parseInteger(value, label, field) {
+  if (typeof value !== "number" || !Number.isInteger(value)) {
+    throw new Error(
+      `local-lab manifest ${label}.${field} must be an integer; received ${describeValue(value)}`
+    );
+  }
+  return value;
+}
+function parseNotes(value, label) {
+  if (!isPlainObject(value)) {
+    throw new Error(
+      `local-lab manifest ${label} must be an object; received ${describeValue(value)}`
+    );
+  }
+  const notes = {};
+  for (const [key, entry] of Object.entries(value)) {
+    notes[key] = entry;
+  }
+  return notes;
+}
+function describeValue(value) {
+  if (value === void 0) {
+    return "undefined";
+  }
+  if (value === null) {
+    return "null";
+  }
+  if (typeof value === "string") {
+    return JSON.stringify(value);
+  }
+  if (typeof value === "number" || typeof value === "boolean") {
+    return String(value);
+  }
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return String(value);
+  }
+}
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+// src/local-lab/resolve-local-lab-profile.ts
+function resolveLocalLabRole(role) {
+  const provider = manifestProviderKindToBuiltIn(role.provider);
+  const baseUrl = normalizeRoleBaseUrl(role.provider, role.baseUrl);
+  const providerConfig = {
+    provider,
+    model: role.model,
+    baseUrl,
+    temperature: role.temperature,
+    seed: role.seed
+  };
+  return {
+    provider: role.provider,
+    baseUrl,
+    model: role.model,
+    ctx: role.ctx,
+    temperature: role.temperature,
+    seed: role.seed,
+    ...role.quantization ? { quantization: role.quantization } : {},
+    providerConfig
+  };
+}
+function normalizeRoleBaseUrl(providerKind, rawBaseUrl) {
+  const trimmed = rawBaseUrl.endsWith("/") ? rawBaseUrl.slice(0, -1) : rawBaseUrl;
+  if (providerKind === "ollama" && !trimmed.endsWith("/api")) {
+    return `${trimmed}/api`;
+  }
+  if (providerKind === "openai-compatible" && !trimmed.endsWith("/v1")) {
+    return `${trimmed}/v1`;
+  }
+  return trimmed;
+}
+function resolveLocalLabProfile(manifest) {
+  return {
+    manifest,
+    responder: resolveLocalLabRole(manifest.responder),
+    judge: resolveLocalLabRole(manifest.judge),
+    ...manifest.embedding ? { embedding: resolveLocalLabRole(manifest.embedding) } : {},
+    phases: manifest.phases,
+    ...manifest.notes ? { notes: manifest.notes } : {}
+  };
+}
+function manifestProviderKindToBuiltIn(kind) {
+  if (kind === "openai-compatible") {
+    return "local-llm";
+  }
+  if (kind === "ollama") {
+    return "ollama";
+  }
+  const exhaustive = kind;
+  throw new Error(`local-lab manifest provider kind unsupported: ${exhaustive}`);
+}
+
+// src/local-lab/preflight.ts
+var DEFAULT_PREFLIGHT_TIMEOUT_MS = 5e3;
+async function preflightLocalLabRole(input, options = {}) {
+  const fetchImpl = options.fetchImpl ?? fetch;
+  const timeoutMs = options.timeoutMs ?? DEFAULT_PREFLIGHT_TIMEOUT_MS;
+  const endpoint = discoveryEndpointFor(input.provider, input.baseUrl);
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(new Error("preflight timeout")), timeoutMs);
+  if (options.signal) {
+    if (options.signal.aborted) {
+      controller.abort(options.signal.reason);
+    } else {
+      options.signal.addEventListener(
+        "abort",
+        () => controller.abort(options.signal.reason),
+        { once: true }
+      );
+    }
+  }
+  const signal = controller.signal;
+  let response;
+  try {
+    response = await fetchImpl(endpoint, {
+      method: "GET",
+      headers: { accept: "application/json" },
+      ...signal ? { signal } : {}
+    });
+  } catch (error) {
+    if (timer) clearTimeout(timer);
+    const detail = error instanceof Error ? error.message : String(error);
+    return {
+      ok: false,
+      provider: input.provider,
+      endpoint,
+      expectedModel: input.model,
+      foundModels: [],
+      expectedCtx: input.ctx,
+      reason: `preflight request to ${endpoint} failed: ${detail}`
+    };
+  }
+  if (!response.ok) {
+    if (timer) clearTimeout(timer);
+    return {
+      ok: false,
+      provider: input.provider,
+      endpoint,
+      expectedModel: input.model,
+      foundModels: [],
+      expectedCtx: input.ctx,
+      reason: `endpoint ${endpoint} returned HTTP ${response.status} ${response.statusText}`
+    };
+  }
+  let parsed;
+  try {
+    parsed = await response.json();
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    if (timer) clearTimeout(timer);
+    return {
+      ok: false,
+      provider: input.provider,
+      endpoint,
+      expectedModel: input.model,
+      foundModels: [],
+      expectedCtx: input.ctx,
+      reason: `endpoint ${endpoint} returned non-JSON body: ${detail}`
+    };
+  }
+  if (timer) clearTimeout(timer);
+  const foundModels = extractDiscoveredModels(input.provider, parsed);
+  const matched = foundModels.find((model) => model.id === input.model);
+  if (matched === void 0) {
+    return {
+      ok: false,
+      provider: input.provider,
+      endpoint,
+      expectedModel: input.model,
+      foundModels,
+      expectedCtx: input.ctx,
+      reason: modelMismatchReason(input.model, foundModels)
+    };
+  }
+  if (matched.contextLength !== void 0 && matched.contextLength < input.ctx) {
+    return {
+      ok: false,
+      provider: input.provider,
+      endpoint,
+      expectedModel: input.model,
+      foundModels,
+      expectedCtx: input.ctx,
+      matchedContextLength: matched.contextLength,
+      reason: `model ${input.model} reports context length ${matched.contextLength} tokens which is below the manifest ctx ${input.ctx}`
+    };
+  }
+  return {
+    ok: true,
+    provider: input.provider,
+    endpoint,
+    expectedModel: input.model,
+    foundModels,
+    ...matched.contextLength !== void 0 ? { matchedContextLength: matched.contextLength } : {}
+  };
+}
+function discoveryEndpointFor(provider, baseUrl) {
+  const trimmed = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
+  if (provider === "openai-compatible") {
+    if (/\/v1$/i.test(trimmed)) {
+      return `${trimmed}/models`;
+    }
+    return `${trimmed}/v1/models`;
+  }
+  if (provider === "ollama") {
+    if (/\/api$/i.test(trimmed)) {
+      return `${trimmed}/tags`;
+    }
+    return `${trimmed}/api/tags`;
+  }
+  const exhaustive = provider;
+  throw new Error(`local-lab preflight provider kind unsupported: ${exhaustive}`);
+}
+function modelMismatchReason(expectedModel, foundModels) {
+  if (foundModels.length === 0) {
+    return `endpoint reported no models; expected ${expectedModel}`;
+  }
+  const foundIds = foundModels.map((model) => model.id);
+  const truncated = foundIds.slice(0, 20);
+  const ellipsis = foundIds.length > truncated.length ? "\u2026" : "";
+  return `endpoint did not report the manifest model ${expectedModel}; found [${truncated.join(", ")}${ellipsis}]`;
+}
+function extractDiscoveredModels(provider, parsed) {
+  if (!isPlainObject2(parsed)) {
+    return [];
+  }
+  if (provider === "openai-compatible") {
+    return extractOpenAiModels(parsed.data);
+  }
+  if (provider === "ollama") {
+    return extractOllamaModels(parsed.models);
+  }
+  const exhaustive = provider;
+  throw new Error(`local-lab preflight provider kind unsupported: ${exhaustive}`);
+}
+function extractOpenAiModels(data) {
+  if (!Array.isArray(data)) {
+    return [];
+  }
+  const models = [];
+  for (const entry of data) {
+    if (!isPlainObject2(entry)) continue;
+    if (typeof entry.id !== "string") continue;
+    models.push({
+      id: entry.id,
+      ...typeof entry.context_length === "number" ? { contextLength: entry.context_length } : {}
+    });
+  }
+  return models;
+}
+function extractOllamaModels(models) {
+  if (!Array.isArray(models)) {
+    return [];
+  }
+  const out = [];
+  for (const entry of models) {
+    if (!isPlainObject2(entry)) continue;
+    if (typeof entry.name !== "string") continue;
+    out.push({ id: entry.name });
+  }
+  return out;
+}
+function isPlainObject2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+// src/local-lab/sequential-phases.ts
+function normalizeBaseUrlForSameness(baseUrl) {
+  return baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
+}
+async function runSequentialPhases(manifest, phases, options = {}) {
+  const outcomes = [];
+  for (let index = 0; index < phases.length; index += 1) {
+    const phase = phases[index];
+    const descriptor = {
+      name: phase.name,
+      role: phase.role
+    };
+    if (index > 0) {
+      const previous = phases[index - 1];
+      const sameEndpoint = normalizeBaseUrlForSameness(phase.role.baseUrl) === normalizeBaseUrlForSameness(previous.role.baseUrl);
+      if (!sameEndpoint) {
+        const manifestNote = readHandoffNote(manifest, previous.name, phase.name);
+        const note = formatHandoffNote(
+          { name: previous.name, role: previous.role },
+          descriptor,
+          manifestNote
+        );
+        options.hooks?.onPhaseHandoff?.(
+          { name: previous.name, role: previous.role },
+          descriptor,
+          note
+        );
+      }
+    }
+    const preflight = await preflightLocalLabRole(
+      {
+        provider: phase.role.provider,
+        baseUrl: phase.role.baseUrl,
+        model: phase.role.model,
+        ctx: phase.role.ctx
+      },
+      options.preflight
+    );
+    if (!preflight.ok) {
+      throw new LocalLabPreflightError({
+        phase: descriptor,
+        preflight,
+        phaseIndex: index
+      });
+    }
+    options.hooks?.onPhasePreflight?.(preflight);
+    options.hooks?.onPhaseStart?.(descriptor);
+    const resolvedRole = resolveLocalLabRole(phase.role);
+    const result = await phase.execute(resolvedRole);
+    const outcome = {
+      phase: descriptor,
+      preflight,
+      result
+    };
+    outcomes.push(outcome);
+    options.hooks?.onPhaseComplete?.(outcome);
+  }
+  return outcomes;
+}
+var LocalLabPreflightError = class extends Error {
+  phase;
+  preflight;
+  phaseIndex;
+  constructor(args) {
+    const reason = args.preflight.ok === false ? args.preflight.reason : "preflight reported success but the scheduler treated it as a failure (internal inconsistency)";
+    super(
+      `local-lab phase "${args.phase.name}" (index ${args.phaseIndex}) preflight failed: ${reason}`
+    );
+    this.name = "LocalLabPreflightError";
+    this.phase = args.phase;
+    this.preflight = args.preflight;
+    this.phaseIndex = args.phaseIndex;
+  }
+};
+function formatHandoffNote(from, to, manifestNote) {
+  if (manifestNote !== void 0 && manifestNote.trim().length > 0) {
+    return manifestNote.trim();
+  }
+  return `stop ${from.name} endpoint, start ${to.name} endpoint at ${to.role.baseUrl} serving model ${to.role.model}, then resume the bench`;
+}
+function readHandoffNote(manifest, from, to) {
+  if (from === "responder" && to === "judge" && typeof manifest.notes?.responderToJudgeHandoff === "string") {
+    return manifest.notes.responderToJudgeHandoff;
+  }
+  return void 0;
+}
+
 // src/runtime-profiles.ts
 var OPENCLAW_REMNIC_PLUGIN_IDS = ["openclaw-remnic", "openclaw-engram"];
 function getOpenClawPluginEntries(raw) {
@@ -10029,6 +10495,9 @@ var codexCliFallbackRegistered = false;
 var codexCliFallbackChain = Promise.resolve();
 async function resolveBenchRuntimeProfile(options) {
   const profile = options.runtimeProfile ?? "baseline";
+  if (profile === "local-lab") {
+    return resolveLocalLabRuntimeProfile(options);
+  }
   const systemProvider = profile === "openclaw-chain" ? null : resolveProviderConfig(
     "system",
     options.systemProvider,
@@ -10205,11 +10674,11 @@ async function resolveBenchRuntimeProfile(options) {
 async function loadRemnicConfigFile(filePath) {
   const parsed = await loadJsonObject(filePath, "Remnic config");
   const remnic = parsed.remnic;
-  if (isPlainObject(remnic)) {
+  if (isPlainObject3(remnic)) {
     return { ...remnic };
   }
   const engram = parsed.engram;
-  if (isPlainObject(engram)) {
+  if (isPlainObject3(engram)) {
     return { ...engram };
   }
   return parsed;
@@ -10220,10 +10689,10 @@ async function loadOpenclawRuntimeConfig(filePath) {
   }
   const parsed = await loadJsonObject(filePath, "OpenClaw config");
   const entry = resolveOpenClawRemnicPluginEntry(parsed);
-  const remnicConfig = isPlainObject(entry?.config) ? { ...entry.config } : {};
+  const remnicConfig = isPlainObject3(entry?.config) ? { ...entry.config } : {};
   const gatewayConfig = {
-    ...isPlainObject(parsed.agents) ? { agents: parsed.agents } : {},
-    ...isPlainObject(parsed.models) ? { models: parsed.models } : {}
+    ...isPlainObject3(parsed.agents) ? { agents: parsed.agents } : {},
+    ...isPlainObject3(parsed.models) ? { models: parsed.models } : {}
   };
   return {
     remnicConfig,
@@ -10240,7 +10709,7 @@ function deriveOpenclawRuntimeContext(configPath) {
   };
 }
 async function loadJsonObject(filePath, label) {
-  const raw = await readFile8(filePath, "utf8");
+  const raw = await readFile9(filePath, "utf8");
   let parsed;
   try {
     parsed = JSON.parse(raw);
@@ -10249,7 +10718,7 @@ async function loadJsonObject(filePath, label) {
       `${label} at ${filePath} contains invalid JSON: ${error instanceof Error ? error.message : String(error)}`
     );
   }
-  if (!isPlainObject(parsed)) {
+  if (!isPlainObject3(parsed)) {
     throw new Error(`${label} at ${filePath} must be a JSON object`);
   }
   return parsed;
@@ -10429,6 +10898,25 @@ function sanitizeProviderConfig(config) {
     ...config.apiKey ? { apiKey: REDACTED_CONFIG_VALUE } : {}
   };
 }
+function applyLocalLabRuntimeOptions(config, options) {
+  const requestTimeout = options.requestTimeout;
+  const max429WaitMs = options.max429WaitMs;
+  const disableThinking = options.disableThinking;
+  const hasRetry = requestTimeout != null || max429WaitMs != null;
+  if (!hasRetry && !disableThinking) {
+    return config;
+  }
+  return {
+    ...config,
+    ...hasRetry ? {
+      retryOptions: {
+        ...requestTimeout != null ? { timeoutMs: requestTimeout } : {},
+        ...max429WaitMs != null ? { max429WaitMs } : {}
+      }
+    } : {},
+    ...disableThinking ? { disableThinking: true } : {}
+  };
+}
 function registerCodexCliFallbackRunnerIfNeeded(config) {
   if (!config || config.provider !== "codex-cli" || codexCliFallbackRegistered) {
     return;
@@ -10524,6 +11012,8 @@ function asProviderFactoryConfig(config) {
     ...config.retryOptions ? { retryOptions: config.retryOptions } : {},
     ...config.disableThinking ? { disableThinking: config.disableThinking } : {},
     ...config.reasoningEffort ? { reasoningEffort: config.reasoningEffort } : {},
+    ...config.temperature !== void 0 ? { temperature: config.temperature } : {},
+    ...config.seed !== void 0 ? { seed: config.seed } : {},
     ...config.responderContextBudgetChars !== void 0 ? { responderContextBudgetChars: config.responderContextBudgetChars } : {},
     ...config.responderPromptBudgetChars !== void 0 ? { responderPromptBudgetChars: config.responderPromptBudgetChars } : {}
   };
@@ -10533,17 +11023,17 @@ function asNonEmptyString(value) {
 }
 function sanitizeGatewayConfig(config) {
   const sanitized = sanitizePersistedConfig(config);
-  return isPlainObject(sanitized) ? sanitized : {};
+  return isPlainObject3(sanitized) ? sanitized : {};
 }
 function sanitizePersistedConfig(config) {
   const sanitized = sanitizePersistedValue(config);
-  return isPlainObject(sanitized) ? sanitized : {};
+  return isPlainObject3(sanitized) ? sanitized : {};
 }
 function sanitizePersistedValue(value) {
   if (Array.isArray(value)) {
     return value.map((entry) => sanitizePersistedValue(entry));
   }
-  if (!isPlainObject(value)) {
+  if (!isPlainObject3(value)) {
     return value;
   }
   const next = {};
@@ -10556,9 +11046,61 @@ function sanitizePersistedValue(value) {
   }
   return next;
 }
-function isPlainObject(value) {
+function isPlainObject3(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
+async function resolveLocalLabRuntimeProfile(options) {
+  if (!options.localLabManifestPath) {
+    throw new Error(
+      "local-lab runtime profile requires localLabManifestPath pointing at a manifest JSON file (issue #1573 PR2)"
+    );
+  }
+  const manifest = await loadLocalLabManifest(options.localLabManifestPath);
+  const resolved = resolveLocalLabProfile(manifest);
+  const drainTimeoutMs = normalizeDrainTimeoutMs2(
+    options.drainTimeout ?? options.requestTimeout
+  );
+  const systemProvider = applyLocalLabRuntimeOptions(
+    sanitizeProviderConfig(resolved.responder.providerConfig),
+    options
+  );
+  const judgeProvider = applyLocalLabRuntimeOptions(
+    sanitizeProviderConfig(resolved.judge.providerConfig),
+    options
+  );
+  const judgeFactoryConfig = judgeProvider ? asProviderFactoryConfig(judgeProvider) : void 0;
+  const judgeProviderInstance = judgeFactoryConfig ? createProvider(judgeFactoryConfig) : void 0;
+  const judge = judgeFactoryConfig ? createProviderBackedJudge(judgeFactoryConfig, judgeProviderInstance) : void 0;
+  const structuredJudge = judgeFactoryConfig ? createProviderBackedStructuredJudge(judgeFactoryConfig, judgeProviderInstance) : void 0;
+  const responderFactoryConfig = systemProvider ? asProviderFactoryConfig(systemProvider) : void 0;
+  const responder = responderFactoryConfig ? createProviderBackedResponder(responderFactoryConfig) : void 0;
+  const lcmObserveConcurrencyOverrides = buildLcmObserveConcurrencyOverrides(options.lcmObserveConcurrency);
+  const baselineConfig = buildBenchBaselineRemnicConfig();
+  const localLabRemnicConfig = {
+    ...baselineConfig,
+    ...lcmObserveConcurrencyOverrides
+  };
+  const effectiveRemnicConfig = withAssistantHooks(
+    localLabRemnicConfig,
+    responder,
+    structuredJudge
+  );
+  return {
+    profile: "local-lab",
+    remnicConfig: sanitizePersistedConfig(localLabRemnicConfig),
+    effectiveRemnicConfig,
+    adapterOptions: {
+      configOverrides: effectiveRemnicConfig,
+      responder,
+      judge,
+      ...drainTimeoutMs ? { drainTimeoutMs } : {}
+    },
+    systemProvider,
+    judgeProvider,
+    internalProvider: null,
+    localLab: resolved
+  };
+}
 
 // src/benchmark.ts
 import fs2 from "fs";
@@ -10570,7 +11112,7 @@ import { expandTildePath as expandTildePath3 } from "@remnic/core";
 import { createHash as createHash6, randomBytes as randomBytes2 } from "crypto";
 import {
   mkdir as mkdir8,
-  readFile as readFile9,
+  readFile as readFile10,
   rename as rename2,
   rm as rm3,
   writeFile as writeFile8
@@ -10631,7 +11173,7 @@ var JudgeCache = class {
     const filePath = this.entryPath(key);
     let raw;
     try {
-      raw = await readFile9(filePath, "utf8");
+      raw = await readFile10(filePath, "utf8");
     } catch {
       return void 0;
     }
@@ -10869,7 +11411,7 @@ function isBenchJudgeResult(value) {
 
 // src/benchmarks/published/ama-bench/runner.ts
 import { randomUUID as randomUUID2 } from "crypto";
-import { readFile as readFile10 } from "fs/promises";
+import { readFile as readFile11 } from "fs/promises";
 import path12 from "path";
 
 // src/benchmarks/published/ama-bench/fixture.ts
@@ -11448,7 +11990,7 @@ async function loadDataset(mode, datasetDir, limit) {
     const filePath = path12.join(datasetDir, "open_end_qa_set.jsonl");
     let raw;
     try {
-      raw = await readFile10(filePath, "utf8");
+      raw = await readFile11(filePath, "utf8");
     } catch (error) {
       throw new Error(
         `AMA-Bench dataset not found at ${filePath}: ${error instanceof Error ? error.message : String(error)}`
@@ -11740,7 +12282,7 @@ function isValidQaPairs(value) {
 
 // src/benchmarks/published/amemgym/runner.ts
 import { randomUUID as randomUUID3 } from "crypto";
-import { readFile as readFile11 } from "fs/promises";
+import { readFile as readFile12 } from "fs/promises";
 import path13 from "path";
 
 // src/benchmarks/published/amemgym/fixture.ts
@@ -12270,7 +12812,7 @@ async function loadDataset2(mode, datasetDir, limit) {
     const datasetErrors = [];
     for (const filename of DATASET_FILENAMES) {
       try {
-        const raw = await readFile11(path13.join(datasetDir, filename), "utf8");
+        const raw = await readFile12(path13.join(datasetDir, filename), "utf8");
         const parsed = parseDataset(raw, filename, normalizedLimit);
         return ensureDatasetProfiles(parsed);
       } catch (error) {
@@ -12444,7 +12986,7 @@ function normalizeRole(role) {
 
 // src/benchmarks/published/memory-arena/runner.ts
 import { randomUUID as randomUUID4 } from "crypto";
-import { readFile as readFile12, readdir as readdir5, stat as stat3 } from "fs/promises";
+import { readFile as readFile13, readdir as readdir5, stat as stat3 } from "fs/promises";
 import path14 from "path";
 import { expandTildePath as expandTildePath2 } from "@remnic/core";
 
@@ -12772,7 +13314,7 @@ async function loadDataset3(mode, datasetDir, limit) {
       if (remainingLimit2 === 0) {
         break;
       }
-      const raw = await readFile12(path14.join(datasetDir, filename), "utf8");
+      const raw = await readFile13(path14.join(datasetDir, filename), "utf8");
       const parsedTasks = [];
       raw.split("\n").forEach((line, lineIndex) => {
         if (line.trim().length === 0) {
@@ -13108,7 +13650,7 @@ async function loadMemoryArenaWebshopProductCatalog(datasetDir) {
       `MemoryArena WebShop product sidecar is ${sourceStat.size} bytes; provide a compact JSON/JSONL sidecar smaller than ${MEMORY_ARENA_WEBSHOP_PRODUCTS_MAX_BYTES} bytes instead of the full WebShop catalog.`
     );
   }
-  const raw = await readFile12(sourcePath, "utf8");
+  const raw = await readFile13(sourcePath, "utf8");
   const records = parseMemoryArenaWebshopSidecarRecords(raw, sourcePath);
   const byAsin = /* @__PURE__ */ new Map();
   for (const record of records) {
@@ -14557,7 +15099,7 @@ function scoreSubtaskSuccess(scores) {
 import { collectTemporalLexicalCues } from "@remnic/core";
 
 // src/benchmarks/published/dataset-loader.ts
-import { readFile as readFile13 } from "fs/promises";
+import { readFile as readFile14 } from "fs/promises";
 import path15 from "path";
 
 // src/benchmarks/published/longmemeval/fixture.ts
@@ -14664,7 +15206,7 @@ async function loadDataset4(options) {
       const abs = path15.join(options.datasetDir, filename);
       let raw;
       try {
-        raw = await readFile13(abs, "utf8");
+        raw = await readFile14(abs, "utf8");
       } catch (error) {
         errors.push(
           `${filename}: ${error instanceof Error ? error.message : String(error)}`
@@ -17912,7 +18454,7 @@ var StructuredLiteralParser = class {
 
 // src/benchmarks/published/personamem/runner.ts
 import { createHash as createHash7, randomUUID as randomUUID7 } from "crypto";
-import { readFile as readFile14, realpath as realpath4 } from "fs/promises";
+import { readFile as readFile15, realpath as realpath4 } from "fs/promises";
 import path17 from "path";
 
 // src/benchmarks/published/personamem/fixture.ts
@@ -18192,7 +18734,7 @@ async function loadDataset8(mode, datasetDir, limit) {
       const datasetPath = path17.join(datasetDir, relativePath);
       let raw;
       try {
-        raw = await readFile14(datasetPath, "utf8");
+        raw = await readFile15(datasetPath, "utf8");
       } catch (error) {
         datasetErrors.push(
           `${relativePath}: ${error instanceof Error ? error.message : String(error)}`
@@ -18250,7 +18792,7 @@ async function hydrateSample(row, datasetRoot) {
     datasetRoot,
     row.chat_history_32k_link
   );
-  const chatHistoryRaw = await readFile14(chatHistoryPath, "utf8");
+  const chatHistoryRaw = await readFile15(chatHistoryPath, "utf8");
   const chatHistory = parseChatHistory(
     chatHistoryRaw,
     row.chat_history_32k_link
@@ -18716,7 +19258,7 @@ function applyLimit6(items, limit) {
 
 // src/benchmarks/published/membench/runner.ts
 import { randomUUID as randomUUID8 } from "crypto";
-import { readFile as readFile15, readdir as readdir7 } from "fs/promises";
+import { readFile as readFile16, readdir as readdir7 } from "fs/promises";
 import path18 from "path";
 
 // src/benchmarks/published/membench/fixture.ts
@@ -18978,7 +19520,7 @@ async function loadDataset9(mode, datasetDir, limit) {
     let remainingLimit = normalizedLimit;
     for (const filename of filenames) {
       try {
-        const raw = await readFile15(path18.join(datasetDir, filename), "utf8");
+        const raw = await readFile16(path18.join(datasetDir, filename), "utf8");
         const parsed = filename.endsWith(".jsonl") ? parseJsonlDataset(raw, filename) : parseJsonDataset(raw, filename);
         const limitedCases = remainingLimit === 0 ? [] : applyLimit7(parsed, remainingLimit);
         if (limitedCases.length > 0) {
@@ -19058,7 +19600,7 @@ function parseJsonlDataset(raw, filename) {
   return cases;
 }
 function parseCase(entry, location) {
-  if (!isPlainObject2(entry)) {
+  if (!isPlainObject4(entry)) {
     throw new Error(`MemBench case ${location} must be an object.`);
   }
   const {
@@ -19177,7 +19719,7 @@ function normalizePublishedNode(node, hints, location) {
       (entry, index) => normalizePublishedNode(entry, hints, `${location}[${index}]`)
     );
   }
-  if (!isPlainObject2(node)) {
+  if (!isPlainObject4(node)) {
     return [];
   }
   const flatCase = normalizeFlatCase(node, hints, location);
@@ -19223,7 +19765,7 @@ function normalizeFlatCase(record, hints, location) {
 function normalizeTrajectoryQaRecord(record, hints, location) {
   const trajectory = record.trajectory ?? record.message_list ?? record.messages;
   const rawQa = record.qa ?? record.QA ?? record.qas ?? record.qa_pairs ?? record.question_answers;
-  const qa = Array.isArray(rawQa) ? rawQa : isPlainObject2(rawQa) ? [rawQa] : void 0;
+  const qa = Array.isArray(rawQa) ? rawQa : isPlainObject4(rawQa) ? [rawQa] : void 0;
   if (!Array.isArray(trajectory) || !Array.isArray(qa) || qa.length === 0) {
     return [];
   }
@@ -19311,7 +19853,7 @@ function appendTrajectoryTurn(turn, _location, coordinate, turns, coordinateInde
     turns.push({ role: "user", content: turn });
     return true;
   }
-  if (!isPlainObject2(turn)) {
+  if (!isPlainObject4(turn)) {
     return false;
   }
   const directMessage = parseDirectMessageTurn(turn);
@@ -19361,7 +19903,7 @@ function normalizeQaPairs(qa, location, coordinateIndex) {
   const pairs = [];
   for (let index = 0; index < qa.length; index += 1) {
     const item = qa[index];
-    if (!isPlainObject2(item)) {
+    if (!isPlainObject4(item)) {
       continue;
     }
     const question = firstString(item.question, item.query, item.prompt);
@@ -19501,7 +20043,7 @@ function firstString(...values) {
   return null;
 }
 function parseTurn(turn, location) {
-  if (!isPlainObject2(turn)) {
+  if (!isPlainObject4(turn)) {
     throw new Error(`MemBench turn ${location} must be an object.`);
   }
   const { role, content } = turn;
@@ -19676,7 +20218,7 @@ function parseChoices(choices, location) {
       D: parsedArray[3]
     };
   }
-  if (!isPlainObject2(choices)) {
+  if (!isPlainObject4(choices)) {
     return void 0;
   }
   const parsed = {
@@ -19839,13 +20381,13 @@ function applyLimit7(items, limit) {
   }
   return items.slice(0, limit);
 }
-function isPlainObject2(value) {
+function isPlainObject4(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
 // src/benchmarks/published/memoryagentbench/runner.ts
 import { randomUUID as randomUUID9 } from "crypto";
-import { access, readFile as readFile16 } from "fs/promises";
+import { access, readFile as readFile17 } from "fs/promises";
 import path19 from "path";
 
 // src/benchmarks/published/memoryagentbench/fixture.ts
@@ -20868,7 +21410,7 @@ async function loadRecSysEntityMapping(datasetDir) {
     }
     let parsed;
     try {
-      parsed = JSON.parse(await readFile16(candidate, "utf8"));
+      parsed = JSON.parse(await readFile17(candidate, "utf8"));
     } catch (error) {
       console.error(
         `  [WARN] MemoryAgentBench ReDial entity mapping ${candidate} is invalid JSON; trying the next candidate: ${error instanceof Error ? error.message : String(error)}`
@@ -21031,7 +21573,7 @@ async function loadDataset10(mode, datasetDir, limit) {
   return ensureDatasetItems(applyLimit8(MEMORY_AGENT_BENCH_SMOKE_FIXTURE, normalizedLimit));
 }
 async function readDatasetFile(filePath, filename) {
-  const raw = await readFile16(filePath, "utf8");
+  const raw = await readFile17(filePath, "utf8");
   const parsed = filename.endsWith(".jsonl") ? parseJsonLines(raw, filename) : parseJsonArray(raw, filename);
   return parsed.map(
     (item, index) => parseMemoryAgentBenchItem(item, `${filename} item ${index + 1}`)
@@ -22644,7 +23186,7 @@ function parseNonNegativeInt(rawValue) {
 
 // src/benchmarks/remnic/page-versioning/runner.ts
 import { randomUUID as randomUUID14 } from "crypto";
-import { mkdir as mkdir9, mkdtemp as mkdtemp4, readFile as readFile17, rm as rm5, writeFile as writeFile9 } from "fs/promises";
+import { mkdir as mkdir9, mkdtemp as mkdtemp4, readFile as readFile18, rm as rm5, writeFile as writeFile9 } from "fs/promises";
 import os6 from "os";
 import path22 from "path";
 import {
@@ -22824,7 +23366,7 @@ async function executeCase2(sample, dependencies) {
         await dependencies.createVersion(pagePath, "modified content", "write", config, void 0, void 0, tmpDir);
         await dependencies.revertToVersion(pagePath, "1", config, void 0, tmpDir);
         const history = await dependencies.listVersions(pagePath, config, tmpDir);
-        const pageContent = await readFile17(pagePath, "utf-8");
+        const pageContent = await readFile18(pagePath, "utf-8");
         const observed = await dependencies.getVersion(pagePath, "3", config, tmpDir);
         return {
           versionIds: history.versions.map((version) => version.versionId),
@@ -22841,7 +23383,7 @@ async function executeCase2(sample, dependencies) {
           await dependencies.createVersion(pagePath, content, "write", pruningConfig, void 0, void 0, tmpDir);
         }
         const history = await dependencies.listVersions(pagePath, pruningConfig, tmpDir);
-        const pageContent = await readFile17(pagePath, "utf-8");
+        const pageContent = await readFile18(pagePath, "utf-8");
         const prunedIds = [];
         for (const versionId of ["1", "2"]) {
           try {
@@ -22882,7 +23424,7 @@ async function executeCase2(sample, dependencies) {
           tmpDir
         );
         const history = await dependencies.listVersions(pagePath, config, tmpDir);
-        const pageContent = await readFile17(pagePath, "utf-8");
+        const pageContent = await readFile18(pagePath, "utf-8");
         const diff = await dependencies.diffVersions(pagePath, "1", "2", config, tmpDir);
         const observedLines = normalizeDiffChangedLines(diff);
         return {
@@ -30099,7 +30641,7 @@ function getBenchmarkLowerIsBetter(benchmarkId) {
 }
 
 // src/integrity/sealed-qrels.ts
-import { readFile as readFile18 } from "fs/promises";
+import { readFile as readFile19 } from "fs/promises";
 function isSealedQrelsArtifact(value) {
   if (!value || typeof value !== "object") {
     return false;
@@ -30169,7 +30711,7 @@ function parseSealedQrels(raw, options = {}) {
   };
 }
 async function loadSealedQrels(filePath, options = {}) {
-  const raw = await readFile18(filePath, "utf8");
+  const raw = await readFile19(filePath, "utf8");
   return parseSealedQrels(raw, options);
 }
 function serializeSealedQrels(artifact) {
@@ -30289,7 +30831,7 @@ function selectFixtureVariant(variants, seed) {
 }
 
 // src/benchmarks/custom/loader.ts
-import { readFile as readFile19 } from "fs/promises";
+import { readFile as readFile20 } from "fs/promises";
 import { parse as parseYaml } from "yaml";
 var CUSTOM_SCORING_VALUES = /* @__PURE__ */ new Set([
   "exact_match",
@@ -30309,7 +30851,7 @@ function parseCustomBenchmark(source) {
 async function loadCustomBenchmarkFile(filePath) {
   let source;
   try {
-    source = await readFile19(filePath, "utf8");
+    source = await readFile20(filePath, "utf8");
   } catch (error) {
     throw new Error(
       `Failed to read custom benchmark file ${filePath}: ${formatError(error)}`
@@ -31501,7 +32043,7 @@ var chatFixture = {
 };
 
 // src/benchmarks/remnic/procedural-recall/ablation.ts
-import { mkdir as mkdir16, mkdtemp as mkdtemp11, rm as rm13, writeFile as writeFile15, readFile as readFile20 } from "fs/promises";
+import { mkdir as mkdir16, mkdtemp as mkdtemp11, rm as rm13, writeFile as writeFile15, readFile as readFile21 } from "fs/promises";
 import os9 from "os";
 import path34 from "path";
 import {
@@ -31624,7 +32166,7 @@ async function runProceduralAblation(options) {
   };
 }
 async function loadAblationFixture(fixturePath) {
-  const raw = await readFile20(fixturePath, "utf8");
+  const raw = await readFile21(fixturePath, "utf8");
   let parsed;
   try {
     parsed = JSON.parse(raw);
@@ -32816,8 +33358,10 @@ export {
   INTEGRITY_CIPHER_ALGORITHM,
   INTEGRITY_HASH_ALGORITHM,
   INTEGRITY_META_FIELDS,
+  LOCAL_LAB_PROVIDER_KINDS,
   LOCOMO_DATASET_FILENAMES,
   LONG_MEM_EVAL_DATASET_FILENAMES,
+  LocalLabPreflightError,
   MEMORY_EVAL_DIMENSIONS,
   MEMORY_EVAL_PUBLIC_LINE,
   MITIGATED_BASELINE_SCENARIOS,
@@ -32896,12 +33440,14 @@ export {
   defaultBenchmarkPublishPath,
   deleteBenchmarkResults,
   discoverAllProviders,
+  discoveryEndpointFor,
   emailFixture,
   entityRecall,
   exactMatch,
   extractMarkdownSectionsByTitle,
   f1Score,
   fixtureToAblationScenarios,
+  formatHandoffNote,
   formatMissingDatasetError,
   generateReport,
   getBenchmark,
@@ -32935,6 +33481,7 @@ export {
   loadBenchmarkResult,
   loadCustomBenchmarkFile,
   loadLoCoMo10,
+  loadLocalLabManifest,
   loadLongMemEvalS,
   loadSealKeyFromEnv,
   loadSealedQrels,
@@ -32946,9 +33493,11 @@ export {
   pairedDeltaConfidenceInterval,
   parseBenchmarkArtifact,
   parseCustomBenchmark,
+  parseLocalLabManifest,
   parseRubricResponse,
   parseSealedQrels,
   precisionAtK,
+  preflightLocalLabRole,
   projectFolderFixture,
   recallAtK,
   redactBenchmarkResultSecrets,
@@ -32965,6 +33514,8 @@ export {
   resolveBenchmarkProgressLogging,
   resolveBenchmarkResultReference,
   resolveBenchmarkRunCount,
+  resolveLocalLabProfile,
+  resolveLocalLabRole,
   resolveStructuredJudge,
   rotateDistractors,
   rougeL,
@@ -32983,6 +33534,7 @@ export {
   runProceduralAblation,
   runProceduralAblationCli,
   runSealedJudge,
+  runSequentialPhases,
   safeHexEqual,
   saveBaseline,
   saveBenchmarkBaseline,