@openlines/opl-pi-sht 0.2.11 → 0.3.0

package/extensions/opl-simplebench/benchmark.ts

@@ -1,5 +1,7 @@
 import * as crypto from "node:crypto";
 import * as fs from "node:fs";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
 import { section, ok, fail, warn, info, msHuman, truncate, sanitizeForReport } from "./util/format";
 import { getOllamaBaseUrl, detectProvider } from "./util/providers";
 import { debugLog } from "./util/debug";
@@ -60,6 +62,92 @@ export function openAiThinkingOptions(thinkingMax: boolean): Record<string, stri
   return thinkingMax ? { reasoning_effort: "max" } : {};
 }
 
+const execFileAsync = promisify(execFile);
+
+/** Credentials for the Bedrock Converse API. */
+export interface AwsCredentials {
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken?: string;
+}
+
+export interface AwsCliCommand { file: string; args: string[] }
+export type AwsCliRunner = (command: AwsCliCommand) => Promise<{ stdout: string }>;
+
+/**
+ * A profile name reaches the AWS CLI as one argv element, never as part of a
+ * shell string. Anything that is not a plain profile name is refused before a
+ * process is spawned: metacharacters would only matter if the value were
+ * interpolated into a shell command, and a leading dash the CLI would read as
+ * another option.
+ */
+const AWS_PROFILE_RE = /^[A-Za-z0-9][A-Za-z0-9_.:@+-]{0,127}$/;
+
+export function isValidAwsProfileName(profile: string): boolean {
+  return AWS_PROFILE_RE.test(profile);
+}
+
+export function awsCredentialsCommand(profile: string): AwsCliCommand {
+  return { file: "aws", args: ["configure", "export-credentials", "--profile", profile] };
+}
+
+const defaultAwsCliRunner: AwsCliRunner = async (command) => {
+  const { stdout } = await execFileAsync(command.file, command.args, { timeout: 15_000, encoding: "utf8" });
+  return { stdout };
+};
+
+function parseAwsCredentials(stdout: string): AwsCredentials | null {
+  try {
+    const parsed = JSON.parse(stdout);
+    if (parsed?.AccessKeyId && parsed?.SecretAccessKey) {
+      return { accessKeyId: parsed.AccessKeyId, secretAccessKey: parsed.SecretAccessKey, sessionToken: parsed.SessionToken };
+    }
+  } catch { /* not JSON: missing CLI, unresolvable profile, or a stray warning line */ }
+  return null;
+}
+
+let _awsCredsCache: AwsCredentials | null | undefined;
+
+/**
+ * Resolve AWS credentials: environment first, then the AWS CLI
+ * (`aws configure export-credentials`), which handles static keys, assume-role
+ * chains (source_profile + role_arn), and SSO. The CLI runs from argv and off
+ * the event loop, so a slow credential chain cannot stall the TUI. Injecting
+ * `env` or `runner` bypasses the per-process cache, which keeps tests from
+ * poisoning the real path.
+ * # ponytail: requires AWS CLI v2; cached per process
+ */
+export async function resolveAwsCredentials(
+  source: { env?: Record<string, string | undefined>; runner?: AwsCliRunner } = {},
+): Promise<AwsCredentials | null> {
+  const env = source.env ?? process.env;
+  const runner = source.runner ?? defaultAwsCliRunner;
+  const cached = source.env === undefined && source.runner === undefined;
+  if (cached && _awsCredsCache !== undefined) return _awsCredsCache;
+
+  const lookup = async (): Promise<AwsCredentials | null> => {
+    if (env.AWS_ACCESS_KEY_ID && env.AWS_SECRET_ACCESS_KEY) {
+      return { accessKeyId: env.AWS_ACCESS_KEY_ID, secretAccessKey: env.AWS_SECRET_ACCESS_KEY, sessionToken: env.AWS_SESSION_TOKEN };
+    }
+    const profile = env.AWS_PROFILE || "default";
+    if (!isValidAwsProfileName(profile)) return null;
+    try {
+      return parseAwsCredentials((await runner(awsCredentialsCommand(profile))).stdout);
+    } catch {
+      return null; /* aws CLI missing or profile unresolvable */
+    }
+  };
+
+  const creds = await lookup();
+  if (cached) _awsCredsCache = creds;
+  return creds;
+}
+
+/** Drop the per-process credential cache (tests, credential rotation). */
+export function resetAwsCredentialsCacheForTest(): void {
+  _awsCredsCache = undefined;
+}
+
 export function createBenchmark() {
 // Use effective config (user overrides merged with defaults)
 const effectiveConfig = getEffectiveConfig();
@@ -174,34 +262,6 @@ function makeOpenAiChatFn(baseUrl: string, apiKey?: string, thinkingMax = false)
   };
 }
 
-/**
- * Resolve AWS credentials: env vars first, then the AWS CLI
- * (`aws configure export-credentials`), which handles static keys,
- * assume-role chains (source_profile + role_arn), and SSO.
- * # ponytail: requires AWS CLI v2; cached per process
- */
-let _awsCredsCache: { accessKeyId: string; secretAccessKey: string; sessionToken?: string } | null | undefined;
-function resolveAwsCredentials(): { accessKeyId: string; secretAccessKey: string; sessionToken?: string } | null {
-  if (_awsCredsCache !== undefined) return _awsCredsCache;
-  if (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) {
-    _awsCredsCache = { accessKeyId: process.env.AWS_ACCESS_KEY_ID, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, sessionToken: process.env.AWS_SESSION_TOKEN };
-    return _awsCredsCache;
-  }
-  try {
-    const prof = process.env.AWS_PROFILE || "default";
-    const out = require("node:child_process")
-      .execSync(`aws configure export-credentials --profile ${prof}`, { timeout: 15000 })
-      .toString();
-    const c = JSON.parse(out);
-    if (c.AccessKeyId && c.SecretAccessKey) {
-      _awsCredsCache = { accessKeyId: c.AccessKeyId, secretAccessKey: c.SecretAccessKey, sessionToken: c.SessionToken };
-      return _awsCredsCache;
-    }
-  } catch { /* aws CLI missing or profile unresolvable */ }
-  _awsCredsCache = null;
-  return null;
-}
-
 /** Minimal SigV4 signer for a POST to the Bedrock Converse API. */
 function sigv4Headers(
   creds: { accessKeyId: string; secretAccessKey: string; sessionToken?: string },
@@ -241,7 +301,7 @@ function sigv4Headers(
  */
 function makeBedrockChatFn(providerInfo: { region?: string }): ChatFn {
   return async (model, messages, options) => {
-    const creds = resolveAwsCredentials();
+    const creds = await resolveAwsCredentials();
     if (!creds) throw new Error("AWS credentials not found — set AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY or ~/.aws/credentials");
     const region = providerInfo.region || process.env.AWS_REGION || "us-east-1";
     const host = `bedrock-runtime.${region}.amazonaws.com`;
@@ -570,8 +630,7 @@ type ProgressFn = (msg: string) => void;
 async function testReasoningExtended(chatFn: ChatFn, model: string, onProgress?: ProgressFn): Promise<{ score: string; scores: string[]; answers: string[]; results: ReasoningTestResult[] }> {
   const results: ReasoningTestResult[] = [];
   const total = REASONING_TESTS.length;
-  for (let i = 0; i < total; i++) {
-    const test = REASONING_TESTS[i];
+  for (const [i, test] of REASONING_TESTS.entries()) {
     onProgress?.(`[1/3] Reasoning ${i + 1}/${total}: ${test.name} (${test.category})...`);
     try {
       const requestedAt = new Date().toISOString();
@@ -646,8 +705,7 @@ async function testToolUsageExtended(chatFn: ChatFn, model: string, useToolResul
 
 async function testCodingLite(chatFn: ChatFn, model: string, onProgress?: (message: string) => void): Promise<{ results: Awaited<ReturnType<typeof runCodingTask>>[]; passed: number; total: number }> {
   const results = [] as Awaited<ReturnType<typeof runCodingTask>>[];
-  for (let index = 0; index < CODING_LITE_TASKS.length; index += 1) {
-    const task = CODING_LITE_TASKS[index];
+  for (const [index, task] of CODING_LITE_TASKS.entries()) {
     const prefix = `[${index + 1}/${CODING_LITE_TASKS.length}] coding-lite: [${task.id}]`;
     onProgress?.(`${prefix} starting...`);
     const result = await runCodingTask(chatFn, model, task, {
@@ -696,6 +754,9 @@ function getCurrentModel(ctx: any): string | undefined {
 
 async function testModelExtended(model: string, ctx?: any, options: SimplebenchOptions = { allModels: false, writeArtifact: true }): Promise<string> {
   const lines: string[] = [];
+  // The artifact lands in the session directory when Pi invoked the tool. A bare
+  // programmatic run has no session, so the harness directory is all there is.
+  const artifactDir = ctx?.cwd ?? process.cwd();
   const totalStart = Date.now();
   const providerInfo = ctx ? detectProvider(ctx) : { kind: "ollama" as const, name: "ollama" };
   const resolvedModel = resolveBenchmarkModel(ctx, model);
@@ -784,7 +845,7 @@ async function testModelExtended(model: string, ctx?: any, options: SimplebenchO
       let artifactPath: string | null = null;
       const serverStats = await captureServerStats();
       if (options.writeArtifact) {
-        try { artifactPath = writeArtifact({ schemaVersion: 1, benchmark: { name: "opl-simplebench", suite, ...(options.tag ? { tag: options.tag } : {}), model, provider: providerInfo.name, providerKind: providerInfo.kind, thinking: { ...thinking, modelMetadataSource: resolvedModel.source }, startedAt: new Date(totalStart).toISOString(), finishedAt: new Date().toISOString(), wallTimeMs: totalMs, artifactEnabled: true }, tests: codingTestRecords(codingSummary), summary: { coding: { passed: codingSummary.passed, total: codingSummary.total, efficiency: { strong: codingSummary.results.filter(r => r.efficiency === "STRONG").length, moderate: codingSummary.results.filter(r => r.efficiency === "MODERATE").length, weak: codingSummary.results.filter(r => r.efficiency === "WEAK").length, fail: codingSummary.results.filter(r => r.efficiency === "FAIL").length } }, ...(serverStats ? { serverStats } : {}) } }); }
+        try { artifactPath = writeArtifact({ schemaVersion: 1, benchmark: { name: "opl-simplebench", suite, ...(options.tag ? { tag: options.tag } : {}), model, provider: providerInfo.name, providerKind: providerInfo.kind, thinking: { ...thinking, modelMetadataSource: resolvedModel.source }, startedAt: new Date(totalStart).toISOString(), finishedAt: new Date().toISOString(), wallTimeMs: totalMs, artifactEnabled: true }, tests: codingTestRecords(codingSummary), summary: { coding: { passed: codingSummary.passed, total: codingSummary.total, efficiency: { strong: codingSummary.results.filter(r => r.efficiency === "STRONG").length, moderate: codingSummary.results.filter(r => r.efficiency === "MODERATE").length, weak: codingSummary.results.filter(r => r.efficiency === "WEAK").length, fail: codingSummary.results.filter(r => r.efficiency === "FAIL").length } }, ...(serverStats ? { serverStats } : {}) } }, artifactDir); }
         catch (e: any) { lines.push(warn(`Artifact could not be written: ${e?.message || e}`)); }
       }
       const effCounts = { strong: codingSummary.results.filter(r => r.efficiency === "STRONG").length, moderate: codingSummary.results.filter(r => r.efficiency === "MODERATE").length, weak: codingSummary.results.filter(r => r.efficiency === "WEAK").length };
@@ -855,7 +916,7 @@ async function testModelExtended(model: string, ctx?: any, options: SimplebenchO
   if (options.writeArtifact) {
     try {
       const artifact = { schemaVersion: 1 as const, benchmark: { name: "opl-simplebench" as const, suite, ...(options.tag ? { tag: options.tag } : {}), model, provider: providerInfo.name, providerKind: providerInfo.kind, thinking: { ...thinking, modelMetadataSource: resolvedModel.source }, startedAt: new Date(totalStart).toISOString(), finishedAt: new Date().toISOString(), wallTimeMs: totalMs, artifactEnabled: true }, tests: artifactTests, summary: { reasoning: { score: reasoning.score, passed: reasoning.results.filter(r => r.pass).length, total: reasoning.results.length }, instructions: instructions.score, tools: tools.score, ...(codingSummary ? { coding: { passed: codingSummary.passed, total: codingSummary.total, efficiency: { strong: codingSummary.results.filter(r => r.efficiency === "STRONG").length, moderate: codingSummary.results.filter(r => r.efficiency === "MODERATE").length, weak: codingSummary.results.filter(r => r.efficiency === "WEAK").length, fail: codingSummary.results.filter(r => r.efficiency === "FAIL").length } } } : {}), ...(researchGrounded ? { researchGrounded: researchGrounded.score } : {}), ...(researchArtifact ? { researchLive: researchArtifact.score } : {}), metrics: aggregate, ...(serverStats ? { serverStats } : {}) } };
-      artifactPath = researchGrounded ? writeArtifactBundle(artifact, researchGrounded.files) : writeArtifact(artifact);
+      artifactPath = researchGrounded ? writeArtifactBundle(artifact, researchGrounded.files, artifactDir) : writeArtifact(artifact, artifactDir);
     } catch (e: any) { lines.push(warn(`Artifact could not be written: ${e?.message || e}`)); }
   }
   

package/extensions/opl-simplebench/coding.ts

@@ -2,6 +2,7 @@ import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import { spawnSync } from "node:child_process";
+import { scrubbedEnv } from "./util/exec-env";
 import { buildToolResultMessages, type ChatFn, type ChatMessage } from "./util/config";
 import { mergeRequestMetrics, metricsFromChat } from "./metrics";
 import type { RequestMetrics } from "./types";
@@ -29,7 +30,7 @@ export interface CodingTaskResult {
   outputTokens: number | null;
   error: string | null;
   metrics: RequestMetrics;
-  /** STRONG = solved in 1 turn; MODERATE = 2–3 turns; WEAK = 4–5 turns; FAIL = not solved. */
+  /** Efficiency grade: STRONG = 1-2 turns, MODERATE = 3-4, WEAK = the 5-turn budget, FAIL = not solved. */
   efficiency: "STRONG" | "MODERATE" | "WEAK" | "FAIL";
 }
 
@@ -105,7 +106,9 @@ export function createCodingTaskDir(task: CodingTaskFixture): string {
 }
 
 export function runCodingVerifier(task: CodingTaskFixture, root: string, mode: "public" | "hidden") {
-  const result = spawnSync(process.execPath, ["--eval", task.verify(root, mode === "hidden")], { cwd: root, encoding: "utf8", timeout: 10_000 });
+  // The verifier evaluates model-authored code, so it runs with an allowlisted
+  // environment (see util/exec-env.ts). Anything it spawns inherits that.
+  const result = spawnSync(process.execPath, ["--eval", task.verify(root, mode === "hidden")], { cwd: root, encoding: "utf8", timeout: 10_000, env: scrubbedEnv() });
   const rawOutput = `${result.stdout ?? ""}${result.stderr ?? ""}`.trim();
   return { passed: result.status === 0, output: result.status === 0 ? rawOutput : truncateVerifierError(rawOutput) };
 }
@@ -175,8 +178,10 @@ async function runSingleShotCodingTask(chatFn: ChatFn, model: string, task: Codi
     ]);
     const raw = response.content.trim();
     const fenced = raw.match(/```(?:[a-z]*)\n([\s\S]*?)```/);
-    const code = fenced ? fenced[1].trim() : raw;
-    const target = resolveCodingPath(root, task.allowedFiles[0]);
+    const code = fenced ? (fenced[1] ?? "").trim() : raw;
+    const firstAllowed = task.allowedFiles[0];
+    if (!firstAllowed) throw new Error(`coding task ${task.id} declares no allowed file`);
+    const target = resolveCodingPath(root, firstAllowed);
     fs.mkdirSync(path.dirname(target), { recursive: true });
     fs.writeFileSync(target, code + "\n");
     const publicResult = runCodingVerifier(task, root, "public");

package/extensions/opl-simplebench/index.ts

@@ -1,7 +1,9 @@
 import type { ExtensionAPI, AgentToolResult } from "@earendil-works/pi-coding-agent";
+import { existsSync, unlinkSync } from "node:fs";
+import { Type } from "typebox";
 import { debugLog } from "./util/debug";
 import { detectProvider } from "./util/providers";
-import { readTestConfig, TOOL_SUPPORT_CACHE_PATH, type ModelTestUserConfig, type RunSequenceProfile } from "./util/config";
+import { readTestConfig, toolSupportCachePath, type ModelTestUserConfig, type RunSequenceProfile } from "./util/config";
 import type { SimplebenchOptions } from "./types";
 import { createBenchmark } from "./benchmark";
 
@@ -107,9 +109,8 @@ pi.registerCommand("simplebench", {
 
     if (arg === "--clear-cache") {
       try {
-        const fs = require("node:fs");
-        if (fs.existsSync(TOOL_SUPPORT_CACHE_PATH)) {
-          fs.unlinkSync(TOOL_SUPPORT_CACHE_PATH);
+        if (existsSync(toolSupportCachePath())) {
+          unlinkSync(toolSupportCachePath());
           ctx.ui.notify("Tool support cache cleared successfully", "info");
         } else {
           ctx.ui.notify("No cache file found to clear", "info");
@@ -141,7 +142,9 @@ pi.registerCommand("simplebench", {
           ctx.ui.notify(`Cooling down ${resolved.pauseMs / 1000}s before iteration ${i + 1}/${resolved.runs.length}...`, "info");
           await new Promise(resolve => setTimeout(resolve, resolved.pauseMs));
         }
-        const { entry, options } = resolved.runs[i];
+        const run = resolved.runs[i];
+        if (!run) continue;
+        const { entry, options } = run;
         const model = options.model || sequenceModel;
         ctx.ui.notify(`Iteration ${i + 1}/${resolved.runs.length} (${entry}) on ${model}...`, "info");
         try {
@@ -236,20 +239,17 @@ pi.registerTool({
   promptGuidelines: [
     "When the user asks to test or evaluate a model, call simplebench with the model name.",
   ],
-  parameters: {
-    type: "object",
-    properties: {
-      model: { type: "string", description: "Model name to test. If omitted, tests the current model." },
-      no_artifact: { type: "boolean", description: "If true, do not write the detailed JSON audit artifact to the current working directory." },
-      thinking_max: { type: "boolean", description: "Request maximum reasoning on an OpenAI-compatible provider or a direct Bedrock model that advertises max thinking. Omit to use provider defaults." },
-      coding_lite: { type: "boolean", description: "Run only the six execution-backed coding tasks in disposable directories." },
-      test_all: { type: "boolean", description: "Run the existing baseline, coding-lite, and deterministic grounded research tests." },
-      research_live: { type: "boolean", description: "Run configured live-search research as an integration smoke test; it does not affect recommendation." },
-      llama_server: { type: "boolean", description: "Capture /props and /metrics from configured llamaServerUrl. Inference routing is unchanged." },
-      llamagputop: { type: "boolean", description: "Capture configured llamagputopUrl /stats. The declared endpoint is authoritative; no Pi model match is required." },
-      tag: { type: "string", description: "Optional single-word label (letters, digits, dot, dash, underscore). Stored under benchmark.tag and prefixed onto the artifact file or bundle name." },
-    },
-  } as any,
+  parameters: Type.Object({
+    model: Type.Optional(Type.String({ description: "Model name to test. If omitted, tests the current model." })),
+    no_artifact: Type.Optional(Type.Boolean({ description: "If true, do not write the detailed JSON audit artifact to the session directory." })),
+    thinking_max: Type.Optional(Type.Boolean({ description: "Request maximum reasoning on an OpenAI-compatible provider or a direct Bedrock model that advertises max thinking. Omit to use provider defaults." })),
+    coding_lite: Type.Optional(Type.Boolean({ description: "Run only the six execution-backed coding tasks in disposable directories." })),
+    test_all: Type.Optional(Type.Boolean({ description: "Run the existing baseline, coding-lite, and deterministic grounded research tests." })),
+    research_live: Type.Optional(Type.Boolean({ description: "Run configured live-search research as an integration smoke test; it does not affect recommendation." })),
+    llama_server: Type.Optional(Type.Boolean({ description: "Capture /props and /metrics from configured llamaServerUrl. Inference routing is unchanged." })),
+    llamagputop: Type.Optional(Type.Boolean({ description: "Capture configured llamagputopUrl /stats. The declared endpoint is authoritative; no Pi model match is required." })),
+    tag: Type.Optional(Type.String({ description: "Optional single-word label (letters, digits, dot, dash, underscore). Stored under benchmark.tag and prefixed onto the artifact file or bundle name." })),
+  }),
   execute: async (_toolCallId, _params, _signal, _onUpdate, ctx) => {
     const params = _params as any;
     if (params?.tag !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(String(params.tag))) {

package/extensions/opl-simplebench/llama-server.ts

@@ -115,14 +115,15 @@ export function parsePrometheusMetrics(text: string): Record<string, number> {
     if (!match) continue;
     const value = Number(match[2]);
     // llama.cpp namespaces every metric as `llamacpp:<name>`; key on the bare name.
-    if (Number.isFinite(value)) out[match[1].replace(/^llamacpp:/, "")] = value;
+    const name = match[1];
+    if (name && Number.isFinite(value)) out[name.replace(/^llamacpp:/, "")] = value;
   }
   return out;
 }
 
 export function diffPrometheusMetrics(before: Record<string, number>, after: Record<string, number>): Record<string, number> {
   const out: Record<string, number> = {};
-  for (const key of Object.keys(after)) out[key] = after[key] - (before[key] ?? 0);
+  for (const key of Object.keys(after)) out[key] = (after[key] ?? 0) - (before[key] ?? 0);
   return out;
 }
 

package/extensions/opl-simplebench/report.ts

@@ -10,12 +10,6 @@ export function formatInstructionScore(instructions: { pass: boolean; score: str
 export function formatTestSummary(tests: Array<{ name: string; pass: boolean; score: string }>, totalMs: number): string[] {
   return [section("SUMMARY"), ...tests.map(t => t.pass ? ok(`${t.name}: ${t.score}`) : fail(`${t.name}: ${t.score}`)), info(`Total time: ${msHuman(totalMs)}`), info(`Score: ${tests.filter(t => t.pass).length}/${tests.length} tests passed`)];
 }
-/** Legacy-compatible category recommendation. New runner uses recommendation() above. */
-export function formatRecommendation(model: string, passed: number, total: number): string[] {
-  const label = passed === total ? "STRONG" : passed >= total - 1 ? "GOOD" : passed >= total - 2 ? "USABLE" : "WEAK";
-  return [section("RECOMMENDATION"), label === "WEAK" ? fail(`${model} is ${label}`) : ok(`${model} is ${label}`)];
-}
-
 export function recommendation(reasoning: string, instructionsPass: boolean, toolsPass: boolean, coding?: { passed: number; total: number; efficiency?: { strong: number; moderate: number; weak: number; fail: number } }) {
   const reasoningPass = reasoning === "STRONG" || reasoning === "MODERATE";
   // Efficiency-weighted coding pass: STRONG=1.0, MODERATE=0.7, WEAK=0.4, FAIL=0

package/extensions/opl-simplebench/research.ts

@@ -95,15 +95,16 @@ export function verifyGroundedResearch(markdown: string, fixture: GroundedResear
   const findings = markdown.match(/##\s+Findings\s*\n([\s\S]*?)(?=\n##\s|$)/i)?.[1] ?? "";
   const sourceSection = markdown.match(/##\s+Sources\s*\n([\s\S]*?)(?=\n##\s|$)/i)?.[1] ?? "";
   const citedClaims = [...findings.matchAll(/^\s*[-*]\s+(.+?)\s+\[([A-Za-z0-9]+)\]\s*$/gm)];
-  const unknownCitations = citedClaims.map(match => match[2]).filter(id => !knownSources.has(id));
+  const citationIds = citedClaims.map((match) => match[2] ?? "");
+  const unknownCitations = citationIds.filter((id) => !knownSources.has(id));
   const missingClaims: string[] = [];
   const wrongCitations: string[] = [];
   for (const claim of fixture.claims) {
-    const match = citedClaims.find(item => normalizeClaim(item[1]) === normalizeClaim(claim.text));
+    const match = citedClaims.find((item) => normalizeClaim(item[1] ?? "") === normalizeClaim(claim.text));
     if (!match) missingClaims.push(claim.text);
-    else if (match[2] !== claim.sourceId) wrongCitations.push(claim.text);
+    else if ((match[2] ?? "") !== claim.sourceId) wrongCitations.push(claim.text);
   }
-  for (const id of new Set(citedClaims.map(match => match[2]).filter(id => knownSources.has(id)))) {
+  for (const id of new Set(citationIds.filter((id) => knownSources.has(id)))) {
     const source = knownSources.get(id)!;
     if (!new RegExp(`\\[${id}\\]\\(${source.url.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\)`).test(sourceSection)) wrongCitations.push(id);
   }

package/extensions/opl-simplebench/scoring.ts

@@ -6,7 +6,7 @@ const NUMERIC_ANSWER = /^\d+$/;
 
 function lastNumericToken(line: string): string | null {
   const tokens = line.match(/\d+/g);
-  return tokens && tokens.length ? tokens[tokens.length - 1] : null;
+  return tokens?.[tokens.length - 1] ?? null;
 }
 
 export function extractAnswer(response: string, expected?: string): { answer: string; method: string } {
@@ -16,7 +16,9 @@ export function extractAnswer(response: string, expected?: string): { answer: st
   // bare-number remnants that thinking templates leak after the real final line.
   if (expected && NUMERIC_ANSWER.test(expected)) {
     for (let i = lines.length - 1; i >= 0; i -= 1) {
-      const token = lastNumericToken(lines[i]);
+      const line = lines[i];
+      if (line === undefined) continue;
+      const token = lastNumericToken(line);
       if (token === expected) {
         return { answer: token, method: i === lines.length - 1 ? "final-line" : "expected-last-numeric" };
       }

package/extensions/opl-simplebench/util/config.ts

@@ -8,8 +8,8 @@
  */
 
 import * as fs from "node:fs";
-import * as os from "node:os";
 import * as path from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
 
 // ============================================================================
 // Configuration Constants
@@ -59,9 +59,15 @@ export const CONFIG = {
 // User Configuration Overrides
 // ============================================================================
 
-const TEST_CONFIG_DIR = process.env.PI_AGENT_DIR || path.join(os.homedir(), ".pi", "agent");
-export const TEST_CONFIG_PATH = path.join(TEST_CONFIG_DIR, "configs", "opl-simplebench.json");
-const LEGACY_TEST_CONFIG_PATH = path.join(TEST_CONFIG_DIR, "simplebench-config.json");
+/** Resolved per call so a custom `PI_AGENT_DIR` (pi's own agent dir) is honoured. */
+export function configPath(): string {
+  return path.join(getAgentDir(), "configs", "opl-simplebench.json");
+}
+
+/** Pre-rename location, still read when the current path is absent. */
+function legacyConfigPath(): string {
+  return path.join(getAgentDir(), "simplebench-config.json");
+}
 
 /** Shape of the user configuration file. */
 export interface ModelTestUserConfig {
@@ -107,8 +113,8 @@ export interface RunSequenceProfile {
  */
 export function readTestConfig(): ModelTestUserConfig {
   try {
-    const configPath = fs.existsSync(TEST_CONFIG_PATH) ? TEST_CONFIG_PATH : LEGACY_TEST_CONFIG_PATH;
-    if (fs.existsSync(configPath)) return JSON.parse(fs.readFileSync(configPath, "utf-8")) as ModelTestUserConfig;
+    const file = fs.existsSync(configPath()) ? configPath() : legacyConfigPath();
+    if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, "utf-8")) as ModelTestUserConfig;
   } catch { /* config read/parse failure is non-critical — defaults are used */ }
   return {};
 }
@@ -161,8 +167,9 @@ export const WEATHER_TOOL_DEFINITION = {
 // Tool Support Cache (path only — backing file is no longer written)
 // ============================================================================
 
-const TOOL_SUPPORT_CACHE_DIR = path.join(os.homedir(), ".pi", "agent", "cache");
-export const TOOL_SUPPORT_CACHE_PATH = path.join(TOOL_SUPPORT_CACHE_DIR, "tool_support.json");
+export function toolSupportCachePath(): string {
+  return path.join(getAgentDir(), "cache", "tool_support.json");
+}
 
 // ============================================================================
 // ChatFn Abstraction

package/extensions/opl-simplebench/util/exec-env.ts

@@ -0,0 +1,36 @@
+/**
+ * Environment for processes that run model-authored code.
+ *
+ * `coding_lite` and `test_all` write the model's answer to disk and execute it,
+ * so whatever the model wrote runs with this process's parent environment. The
+ * harness is normally started from a shell that holds provider keys, cloud
+ * credentials and agent paths, and none of those belong in front of generated
+ * code. This is therefore an allowlist, not a denylist: anything not listed here
+ * is absent from the child, including every `*_API_KEY`, `AWS_*`, `GH_*` and
+ * `PI_*` variable.
+ *
+ * `NODE_OPTIONS` and `NODE_PATH` are deliberately never forwarded: both can make
+ * Node load attacker-chosen files.
+ */
+export const INHERITED_KEYS = [
+  // POSIX: node resolution, temp files, locale and timezone.
+  "PATH", "HOME", "TMPDIR", "TEMP", "TMP", "LANG", "LC_ALL", "TZ", "SHELL", "USER", "LOGNAME",
+  // TLS trust the child may legitimately need.
+  "SSL_CERT_FILE", "SSL_CERT_DIR", "NODE_EXTRA_CA_CERTS",
+  // Terminal capabilities only; no secrets live here.
+  "TERM", "NO_COLOR", "FORCE_COLOR", "COLORTERM",
+  // Windows: spawning and node resolution fail without these.
+  "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT", "USERPROFILE", "HOMEDRIVE", "HOMEPATH",
+  "APPDATA", "LOCALAPPDATA",
+];
+
+/** Build the environment for a child that executes untrusted, model-authored code. */
+export function scrubbedEnv(base: NodeJS.ProcessEnv = process.env): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const key of INHERITED_KEYS) {
+    const value = base[key];
+    // NUL-containing values are rejected by spawn(); drop them instead of throwing.
+    if (typeof value === "string" && value.length > 0 && !value.includes("\0")) out[key] = value;
+  }
+  return out;
+}

package/extensions/opl-simplebench/util/format.ts

@@ -258,7 +258,7 @@ export function sanitizeForReport(s: string, maxLines = 40): string {
   // Use strict detection: require a closing tag AND a common HTML opening tag to avoid
   // false positives on code that mentions HTML-like syntax (e.g., "use <Item> from 'react'")
   if (/<!DOCTYPE\b|<html[\s>]/i.test(cleaned) || (/<[a-z][\s\S]*>/i.test(cleaned) && cleaned.includes("</") && /<(?:div|span|p|head|body|html|table|form|script)\b/i.test(cleaned))) {
-    const firstLine = cleaned.split("\n")[0];
+    const firstLine = cleaned.split("\n")[0] ?? "";
     return truncate(firstLine, 200) + "\n  ℹ️  (HTML response truncated)";
   }
 
@@ -365,7 +365,7 @@ function parseParamCount(s: string): number | undefined {
   const match = str.match(/^([\d.]+)\s*([bmt]?|a(?:pple)?)$/);
   if (!match) return undefined;
 
-  const num = parseFloat(match[1]);
+  const num = parseFloat(match[1] ?? "");
   if (isNaN(num) || num <= 0) return undefined;
 
   const suffix = match[2];