@openlines/opl-pi-sht 0.2.12 → 0.3.1

package/extensions/opl-simplebench/README.md

@@ -32,7 +32,7 @@ simplebench({ model: "global.openai.gpt-5.6-terra", coding_lite: true })
 simplebench({ model: "global.openai.gpt-5.6-terra", test_all: true })
 ```
 
-`--no-artifact` and `no_artifact: true` suppress the JSON file. Default runs use the provider's sampling and reasoning defaults and write one JSON report per model to Pi's current working directory. `--thinking-max` and `thinking_max: true` request `reasoning_effort: "max"` for OpenAI-compatible providers, or Pi's model-aware Bedrock max-thinking path for direct Bedrock models that advertise `reasoning: true` and `thinkingLevelMap.max`. Other providers reject that mode rather than silently using defaults.
+`--no-artifact` and `no_artifact: true` suppress the JSON file. Default runs use the provider's sampling and reasoning defaults and write one JSON report per model to the current Pi session directory (`ExtensionContext.cwd`). Without an active session context, artifacts fall back to the harness working directory. `--thinking-max` and `thinking_max: true` request `reasoning_effort: "max"` for OpenAI-compatible providers, or Pi's model-aware Bedrock max-thinking path for direct Bedrock models that advertise `reasoning: true` and `thinkingLevelMap.max`. Other providers reject that mode rather than silently using defaults.
 
 ## Extension features
 
@@ -172,7 +172,7 @@ The stats endpoint is metadata-only and does not change inference routing. Its s
 
 ## Artifacts and privacy
 
-Baseline and coding-lite artifacts are written to `process.cwd()` as JSON. `--test-all` writes a bundle:
+Baseline and coding-lite artifacts use the current Pi session directory (`ExtensionContext.cwd`), passed down explicitly; without a Pi session, they fall back to the harness working directory. `--test-all` writes a bundle:
 
 ```text
 simplebench--test-all-<sanitized-model>-<thinking>-<UTC-timestamp>/
@@ -211,6 +211,8 @@ The model receives weather and calculation tools. Simplebench validates expected
 
 Coding-lite is the execution-backed coding-agent suite. It runs model-directed edits in temporary directories. The model can list, search, read, write, and run the fixture's public tests, but cannot execute arbitrary commands, access the real repository, read hidden verification code, or use the network. The host runs hidden verification after the model stops.
 
+**The verifier executes model-authored code.** Verification runs `node --eval` over the model's files, so that process is started with an allowlisted environment (`util/exec-env.ts`): `PATH`, `HOME`, temp-directory and locale variables, TLS trust paths, terminal capability flags, and the Windows equivalents. Everything else is absent — every `*_API_KEY`, `AWS_*`, `GH_*`, `PI_*`, and any other secret your shell happens to export — and `NODE_OPTIONS`/`NODE_PATH` are never forwarded because both can make Node load attacker-chosen files. Anything the model's code spawns inherits that environment, so a nested `node src/cli.mjs` sees the same allowlist. This limits what generated code can *read out of the process*; it is not sandboxing. The child still runs with your user's filesystem permissions and network reach, and a `HOME` it can see points at files it could read by path anyway. Use `--no-artifact` and a dedicated shell if that is not acceptable for your environment.
+
 The agent loop defaults to **5 turns** per task (reduced from 12). This tighter budget forces the model to converge on a solution quickly and makes turn count a meaningful quality signal. Results are scored on three dimensions:
 
 - **STRONG** — solved in 1–2 turns (first or second attempt correct)
@@ -230,17 +232,17 @@ Use `--coding-lite` for coding tasks only. Use `--test-all` to run the existing
 
 ### Recommendation policy
 
-The recommendation is based on the three (or four) displayed categories:
+Four categories can be in play: the closed-answer contract, instruction following, tool usage, and (with `--coding-lite`) coding. A category counts as passed when its own rule succeeds - the closed-answer contract counts as passed at `STRONG` **or** `MODERATE`, and coding counts as passed on its efficiency-weighted average. The label then depends on how many passed, and on how many categories actually ran:
 
-| Result | Recommendation |
-|---|---|
-| Strong reasoning, JSON pass, tool pass, all coding STRONG | STRONG |
-| Strong reasoning, JSON pass, tool pass (coding not all STRONG) | GOOD |
-| Any three categories pass | USABLE |
-| Two categories pass | LIMITED |
-| One or zero categories pass | WEAK |
+| Categories passed | Baseline (3 categories) | With coding (4 categories) |
+|---|---|---|
+| All | STRONG if the contract is `STRONG` and, when coding ran, every task is `STRONG`; GOOD otherwise | |
+| All but one | GOOD | GOOD |
+| Two | USABLE | LIMITED |
+| One | LIMITED | LIMITED |
+| None | WEAK | WEAK |
 
-When coding-lite is included, the coding category uses efficiency-weighted scoring (STRONG=1.0, MODERATE=0.7, WEAK=0.4, FAIL=0). The category counts as passing when the weighted average ≥ 0.5 across all tasks.
+`WEAK` therefore requires that **nothing** passed; one passing category is `LIMITED`. A run that omits coding can report `USABLE` where the same results with coding report `LIMITED`, because the denominator changes. For runs with no clean category counts (an error or a cancelled run), `report.ts` falls back to the average `tests[].score`: `>= 0.8` `GOOD`, `>= 0.5` `USABLE`, `>= 0.2` `LIMITED`, otherwise `WEAK`. The coding category is scored efficiency-weighted (STRONG=1.0, MODERATE=0.7, WEAK=0.4, FAIL=0) and passes when that weighted average is >= 0.5 across all tasks.
 
 ## Provider authentication
 
@@ -260,6 +262,8 @@ Simplebench derives the region from the selected model's Bedrock base URL, signs
 aws configure export-credentials --profile "$AWS_PROFILE"
 ```
 
+The CLI is invoked with an argument vector, never through a shell, so the profile value is never interpolated into a command string; and it runs asynchronously, so a slow SSO or assume-role chain cannot stall the TUI. A profile name that is not a plain token (`^[A-Za-z0-9][A-Za-z0-9_.:@+-]{0,127}$`) is refused before any process is spawned — `AWS_PROFILE` set by repo tooling such as `.envrc` or a Makefile is not trusted to reach the CLI. Set `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` (plus `AWS_SESSION_TOKEN`) to bypass the CLI entirely.
+
 AWS CLI v2 is therefore required when the selected profile uses SSO or an assume-role chain.
 
 ## Troubleshooting
@@ -270,7 +274,7 @@ AWS CLI v2 is therefore required when the selected profile uses SSO or an assume
 | OpenRouter returns 401 | Set `OPENROUTER_API_KEY`, configure the provider key, or authenticate Pi so `auth.json` has an API key. |
 | Bedrock says credentials are unavailable | Run `aws configure export-credentials --profile "$AWS_PROFILE"` and resolve profile login/role issues first. |
 | Bedrock rejects `--thinking-max` | The selected model does not advertise max thinking in Pi metadata, or the provider rejected its model-specific reasoning request. |
-| Artifact is absent | Check the report for `--no-artifact` or an artifact-write warning; the file is written to Pi's cwd. |
+| Artifact is absent | Check the report for `--no-artifact` or an artifact-write warning; the file lands in the session directory, not where the shell was standing. |
 | Token or TTFT values are null | The provider response did not expose authoritative usage/timing values. |
 
 ## Architecture
@@ -279,13 +283,15 @@ AWS CLI v2 is therefore required when the selected profile uses SSO or an assume
 index.ts       Pi registration, command parsing, tool schema
 benchmark.ts   provider adapters and benchmark orchestration
 llama-server.ts direct llama-server /props and /metrics capture
+coding.ts      the six execution-backed coding tasks and their agent loop
+research.ts    grounded research fixtures and the deterministic verifier
 tests.ts       benchmark fixtures
 scoring.ts     extraction and scoring
 metrics.ts     request/usage aggregation
-artifact.ts    cwd JSON artifact writer
+artifact.ts    session-directory JSON artifact writer
 report.ts      terminal output and recommendation policy
 types.ts       shared benchmark types
-util/          local configuration, formatting, debugging, and provider helpers
+util/          local configuration, formatting, debugging, exec-environment scrubbing, and provider helpers
 ```
 
-The extension is self-contained under `extensions/opl-simplebench/`, ready to copy into the `opl-pi-sht` extension stack after local verification.
+Everything lives under `extensions/opl-simplebench/` because `install.sh` installs one extension directory at a time; the only environment boundary model-authored code gets is `util/exec-env.ts`, which is where a new child process must go through.

package/extensions/opl-simplebench/artifact.ts

@@ -1,5 +1,8 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
+
+// The output directory is always passed in: artifacts land in the session directory
+// (`ExtensionContext.cwd`), not wherever the harness was started.
 import type { RunArtifact } from "./types";
 
 export function artifactFileName(model: string, suite: "baseline" | "coding-lite" | "test-all" = "baseline", thinking: "default" | "max" = "default", date = new Date(), tag?: string): string {
@@ -9,10 +12,10 @@ export function artifactFileName(model: string, suite: "baseline" | "coding-lite
   return `simplebench${tag ? `-${tag}-` : "--"}${suiteName}-${safeModel}-${thinking}-${stamp}.json`;
 }
 
-export function writeArtifactBundle(artifact: RunArtifact, files: Record<string, string>): string {
+export function writeArtifactBundle(artifact: RunArtifact, files: Record<string, string>, cwd: string): string {
   const name = artifactFileName(artifact.benchmark.model, artifact.benchmark.suite || "baseline", artifact.benchmark.thinking.requested, new Date(), artifact.benchmark.tag).replace(/\.json$/, "");
   for (let suffix = 0; ; suffix += 1) {
-    const output = path.resolve(process.cwd(), suffix ? `${name}-${suffix}` : name);
+    const output = path.resolve(cwd, suffix ? `${name}-${suffix}` : name);
     try {
       fs.mkdirSync(output);
       fs.writeFileSync(path.join(output, "result.json"), JSON.stringify(artifact, null, 2) + "\n", "utf8");
@@ -24,13 +27,13 @@ export function writeArtifactBundle(artifact: RunArtifact, files: Record<string,
   }
 }
 
-export function writeArtifact(artifact: RunArtifact): string {
+export function writeArtifact(artifact: RunArtifact, cwd: string): string {
   const name = artifactFileName(artifact.benchmark.model, artifact.benchmark.suite || "baseline", artifact.benchmark.thinking.requested, new Date(), artifact.benchmark.tag);
   const extension = path.extname(name);
   const base = name.slice(0, -extension.length);
   const contents = JSON.stringify(artifact, null, 2) + "\n";
   for (let suffix = 0; ; suffix += 1) {
-    const output = path.resolve(process.cwd(), suffix ? `${base}-${suffix}${extension}` : name);
+    const output = path.resolve(cwd, suffix ? `${base}-${suffix}${extension}` : name);
     try {
       fs.writeFileSync(output, contents, { encoding: "utf8", flag: "wx" });
       return output;

package/extensions/opl-simplebench/benchmark.ts

@@ -1,5 +1,7 @@
 import * as crypto from "node:crypto";
 import * as fs from "node:fs";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
 import { section, ok, fail, warn, info, msHuman, truncate, sanitizeForReport } from "./util/format";
 import { getOllamaBaseUrl, detectProvider } from "./util/providers";
 import { debugLog } from "./util/debug";
@@ -60,6 +62,92 @@ export function openAiThinkingOptions(thinkingMax: boolean): Record<string, stri
   return thinkingMax ? { reasoning_effort: "max" } : {};
 }
 
+const execFileAsync = promisify(execFile);
+
+/** Credentials for the Bedrock Converse API. */
+export interface AwsCredentials {
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken?: string;
+}
+
+export interface AwsCliCommand { file: string; args: string[] }
+export type AwsCliRunner = (command: AwsCliCommand) => Promise<{ stdout: string }>;
+
+/**
+ * A profile name reaches the AWS CLI as one argv element, never as part of a
+ * shell string. Anything that is not a plain profile name is refused before a
+ * process is spawned: metacharacters would only matter if the value were
+ * interpolated into a shell command, and a leading dash the CLI would read as
+ * another option.
+ */
+const AWS_PROFILE_RE = /^[A-Za-z0-9][A-Za-z0-9_.:@+-]{0,127}$/;
+
+export function isValidAwsProfileName(profile: string): boolean {
+  return AWS_PROFILE_RE.test(profile);
+}
+
+export function awsCredentialsCommand(profile: string): AwsCliCommand {
+  return { file: "aws", args: ["configure", "export-credentials", "--profile", profile] };
+}
+
+const defaultAwsCliRunner: AwsCliRunner = async (command) => {
+  const { stdout } = await execFileAsync(command.file, command.args, { timeout: 15_000, encoding: "utf8" });
+  return { stdout };
+};
+
+function parseAwsCredentials(stdout: string): AwsCredentials | null {
+  try {
+    const parsed = JSON.parse(stdout);
+    if (parsed?.AccessKeyId && parsed?.SecretAccessKey) {
+      return { accessKeyId: parsed.AccessKeyId, secretAccessKey: parsed.SecretAccessKey, sessionToken: parsed.SessionToken };
+    }
+  } catch { /* not JSON: missing CLI, unresolvable profile, or a stray warning line */ }
+  return null;
+}
+
+let _awsCredsCache: AwsCredentials | null | undefined;
+
+/**
+ * Resolve AWS credentials: environment first, then the AWS CLI
+ * (`aws configure export-credentials`), which handles static keys, assume-role
+ * chains (source_profile + role_arn), and SSO. The CLI runs from argv and off
+ * the event loop, so a slow credential chain cannot stall the TUI. Injecting
+ * `env` or `runner` bypasses the per-process cache, which keeps tests from
+ * poisoning the real path.
+ * # ponytail: requires AWS CLI v2; cached per process
+ */
+export async function resolveAwsCredentials(
+  source: { env?: Record<string, string | undefined>; runner?: AwsCliRunner } = {},
+): Promise<AwsCredentials | null> {
+  const env = source.env ?? process.env;
+  const runner = source.runner ?? defaultAwsCliRunner;
+  const cached = source.env === undefined && source.runner === undefined;
+  if (cached && _awsCredsCache !== undefined) return _awsCredsCache;
+
+  const lookup = async (): Promise<AwsCredentials | null> => {
+    if (env.AWS_ACCESS_KEY_ID && env.AWS_SECRET_ACCESS_KEY) {
+      return { accessKeyId: env.AWS_ACCESS_KEY_ID, secretAccessKey: env.AWS_SECRET_ACCESS_KEY, sessionToken: env.AWS_SESSION_TOKEN };
+    }
+    const profile = env.AWS_PROFILE || "default";
+    if (!isValidAwsProfileName(profile)) return null;
+    try {
+      return parseAwsCredentials((await runner(awsCredentialsCommand(profile))).stdout);
+    } catch {
+      return null; /* aws CLI missing or profile unresolvable */
+    }
+  };
+
+  const creds = await lookup();
+  if (cached) _awsCredsCache = creds;
+  return creds;
+}
+
+/** Drop the per-process credential cache (tests, credential rotation). */
+export function resetAwsCredentialsCacheForTest(): void {
+  _awsCredsCache = undefined;
+}
+
 export function createBenchmark() {
 // Use effective config (user overrides merged with defaults)
 const effectiveConfig = getEffectiveConfig();
@@ -174,34 +262,6 @@ function makeOpenAiChatFn(baseUrl: string, apiKey?: string, thinkingMax = false)
   };
 }
 
-/**
- * Resolve AWS credentials: env vars first, then the AWS CLI
- * (`aws configure export-credentials`), which handles static keys,
- * assume-role chains (source_profile + role_arn), and SSO.
- * # ponytail: requires AWS CLI v2; cached per process
- */
-let _awsCredsCache: { accessKeyId: string; secretAccessKey: string; sessionToken?: string } | null | undefined;
-function resolveAwsCredentials(): { accessKeyId: string; secretAccessKey: string; sessionToken?: string } | null {
-  if (_awsCredsCache !== undefined) return _awsCredsCache;
-  if (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) {
-    _awsCredsCache = { accessKeyId: process.env.AWS_ACCESS_KEY_ID, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, sessionToken: process.env.AWS_SESSION_TOKEN };
-    return _awsCredsCache;
-  }
-  try {
-    const prof = process.env.AWS_PROFILE || "default";
-    const out = require("node:child_process")
-      .execSync(`aws configure export-credentials --profile ${prof}`, { timeout: 15000 })
-      .toString();
-    const c = JSON.parse(out);
-    if (c.AccessKeyId && c.SecretAccessKey) {
-      _awsCredsCache = { accessKeyId: c.AccessKeyId, secretAccessKey: c.SecretAccessKey, sessionToken: c.SessionToken };
-      return _awsCredsCache;
-    }
-  } catch { /* aws CLI missing or profile unresolvable */ }
-  _awsCredsCache = null;
-  return null;
-}
-
 /** Minimal SigV4 signer for a POST to the Bedrock Converse API. */
 function sigv4Headers(
   creds: { accessKeyId: string; secretAccessKey: string; sessionToken?: string },
@@ -241,7 +301,7 @@ function sigv4Headers(
  */
 function makeBedrockChatFn(providerInfo: { region?: string }): ChatFn {
   return async (model, messages, options) => {
-    const creds = resolveAwsCredentials();
+    const creds = await resolveAwsCredentials();
     if (!creds) throw new Error("AWS credentials not found — set AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY or ~/.aws/credentials");
     const region = providerInfo.region || process.env.AWS_REGION || "us-east-1";
     const host = `bedrock-runtime.${region}.amazonaws.com`;
@@ -570,8 +630,7 @@ type ProgressFn = (msg: string) => void;
 async function testReasoningExtended(chatFn: ChatFn, model: string, onProgress?: ProgressFn): Promise<{ score: string; scores: string[]; answers: string[]; results: ReasoningTestResult[] }> {
   const results: ReasoningTestResult[] = [];
   const total = REASONING_TESTS.length;
-  for (let i = 0; i < total; i++) {
-    const test = REASONING_TESTS[i];
+  for (const [i, test] of REASONING_TESTS.entries()) {
     onProgress?.(`[1/3] Reasoning ${i + 1}/${total}: ${test.name} (${test.category})...`);
     try {
       const requestedAt = new Date().toISOString();
@@ -646,8 +705,7 @@ async function testToolUsageExtended(chatFn: ChatFn, model: string, useToolResul
 
 async function testCodingLite(chatFn: ChatFn, model: string, onProgress?: (message: string) => void): Promise<{ results: Awaited<ReturnType<typeof runCodingTask>>[]; passed: number; total: number }> {
   const results = [] as Awaited<ReturnType<typeof runCodingTask>>[];
-  for (let index = 0; index < CODING_LITE_TASKS.length; index += 1) {
-    const task = CODING_LITE_TASKS[index];
+  for (const [index, task] of CODING_LITE_TASKS.entries()) {
     const prefix = `[${index + 1}/${CODING_LITE_TASKS.length}] coding-lite: [${task.id}]`;
     onProgress?.(`${prefix} starting...`);
     const result = await runCodingTask(chatFn, model, task, {
@@ -696,6 +754,9 @@ function getCurrentModel(ctx: any): string | undefined {
 
 async function testModelExtended(model: string, ctx?: any, options: SimplebenchOptions = { allModels: false, writeArtifact: true }): Promise<string> {
   const lines: string[] = [];
+  // The artifact lands in the session directory when Pi invoked the tool. A bare
+  // programmatic run has no session, so the harness directory is all there is.
+  const artifactDir = ctx?.cwd ?? process.cwd();
   const totalStart = Date.now();
   const providerInfo = ctx ? detectProvider(ctx) : { kind: "ollama" as const, name: "ollama" };
   const resolvedModel = resolveBenchmarkModel(ctx, model);
@@ -784,7 +845,7 @@ async function testModelExtended(model: string, ctx?: any, options: SimplebenchO
       let artifactPath: string | null = null;
       const serverStats = await captureServerStats();
       if (options.writeArtifact) {
-        try { artifactPath = writeArtifact({ schemaVersion: 1, benchmark: { name: "opl-simplebench", suite, ...(options.tag ? { tag: options.tag } : {}), model, provider: providerInfo.name, providerKind: providerInfo.kind, thinking: { ...thinking, modelMetadataSource: resolvedModel.source }, startedAt: new Date(totalStart).toISOString(), finishedAt: new Date().toISOString(), wallTimeMs: totalMs, artifactEnabled: true }, tests: codingTestRecords(codingSummary), summary: { coding: { passed: codingSummary.passed, total: codingSummary.total, efficiency: { strong: codingSummary.results.filter(r => r.efficiency === "STRONG").length, moderate: codingSummary.results.filter(r => r.efficiency === "MODERATE").length, weak: codingSummary.results.filter(r => r.efficiency === "WEAK").length, fail: codingSummary.results.filter(r => r.efficiency === "FAIL").length } }, ...(serverStats ? { serverStats } : {}) } }); }
+        try { artifactPath = writeArtifact({ schemaVersion: 1, benchmark: { name: "opl-simplebench", suite, ...(options.tag ? { tag: options.tag } : {}), model, provider: providerInfo.name, providerKind: providerInfo.kind, thinking: { ...thinking, modelMetadataSource: resolvedModel.source }, startedAt: new Date(totalStart).toISOString(), finishedAt: new Date().toISOString(), wallTimeMs: totalMs, artifactEnabled: true }, tests: codingTestRecords(codingSummary), summary: { coding: { passed: codingSummary.passed, total: codingSummary.total, efficiency: { strong: codingSummary.results.filter(r => r.efficiency === "STRONG").length, moderate: codingSummary.results.filter(r => r.efficiency === "MODERATE").length, weak: codingSummary.results.filter(r => r.efficiency === "WEAK").length, fail: codingSummary.results.filter(r => r.efficiency === "FAIL").length } }, ...(serverStats ? { serverStats } : {}) } }, artifactDir); }
         catch (e: any) { lines.push(warn(`Artifact could not be written: ${e?.message || e}`)); }
       }
       const effCounts = { strong: codingSummary.results.filter(r => r.efficiency === "STRONG").length, moderate: codingSummary.results.filter(r => r.efficiency === "MODERATE").length, weak: codingSummary.results.filter(r => r.efficiency === "WEAK").length };
@@ -855,7 +916,7 @@ async function testModelExtended(model: string, ctx?: any, options: SimplebenchO
   if (options.writeArtifact) {
     try {
       const artifact = { schemaVersion: 1 as const, benchmark: { name: "opl-simplebench" as const, suite, ...(options.tag ? { tag: options.tag } : {}), model, provider: providerInfo.name, providerKind: providerInfo.kind, thinking: { ...thinking, modelMetadataSource: resolvedModel.source }, startedAt: new Date(totalStart).toISOString(), finishedAt: new Date().toISOString(), wallTimeMs: totalMs, artifactEnabled: true }, tests: artifactTests, summary: { reasoning: { score: reasoning.score, passed: reasoning.results.filter(r => r.pass).length, total: reasoning.results.length }, instructions: instructions.score, tools: tools.score, ...(codingSummary ? { coding: { passed: codingSummary.passed, total: codingSummary.total, efficiency: { strong: codingSummary.results.filter(r => r.efficiency === "STRONG").length, moderate: codingSummary.results.filter(r => r.efficiency === "MODERATE").length, weak: codingSummary.results.filter(r => r.efficiency === "WEAK").length, fail: codingSummary.results.filter(r => r.efficiency === "FAIL").length } } } : {}), ...(researchGrounded ? { researchGrounded: researchGrounded.score } : {}), ...(researchArtifact ? { researchLive: researchArtifact.score } : {}), metrics: aggregate, ...(serverStats ? { serverStats } : {}) } };
-      artifactPath = researchGrounded ? writeArtifactBundle(artifact, researchGrounded.files) : writeArtifact(artifact);
+      artifactPath = researchGrounded ? writeArtifactBundle(artifact, researchGrounded.files, artifactDir) : writeArtifact(artifact, artifactDir);
     } catch (e: any) { lines.push(warn(`Artifact could not be written: ${e?.message || e}`)); }
   }
   

package/extensions/opl-simplebench/coding.ts

@@ -2,6 +2,7 @@ import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import { spawnSync } from "node:child_process";
+import { scrubbedEnv } from "./util/exec-env";
 import { buildToolResultMessages, type ChatFn, type ChatMessage } from "./util/config";
 import { mergeRequestMetrics, metricsFromChat } from "./metrics";
 import type { RequestMetrics } from "./types";
@@ -29,7 +30,7 @@ export interface CodingTaskResult {
   outputTokens: number | null;
   error: string | null;
   metrics: RequestMetrics;
-  /** STRONG = solved in 1 turn; MODERATE = 2–3 turns; WEAK = 4–5 turns; FAIL = not solved. */
+  /** Efficiency grade: STRONG = 1-2 turns, MODERATE = 3-4, WEAK = the 5-turn budget, FAIL = not solved. */
   efficiency: "STRONG" | "MODERATE" | "WEAK" | "FAIL";
 }
 
@@ -105,7 +106,9 @@ export function createCodingTaskDir(task: CodingTaskFixture): string {
 }
 
 export function runCodingVerifier(task: CodingTaskFixture, root: string, mode: "public" | "hidden") {
-  const result = spawnSync(process.execPath, ["--eval", task.verify(root, mode === "hidden")], { cwd: root, encoding: "utf8", timeout: 10_000 });
+  // The verifier evaluates model-authored code, so it runs with an allowlisted
+  // environment (see util/exec-env.ts). Anything it spawns inherits that.
+  const result = spawnSync(process.execPath, ["--eval", task.verify(root, mode === "hidden")], { cwd: root, encoding: "utf8", timeout: 10_000, env: scrubbedEnv() });
   const rawOutput = `${result.stdout ?? ""}${result.stderr ?? ""}`.trim();
   return { passed: result.status === 0, output: result.status === 0 ? rawOutput : truncateVerifierError(rawOutput) };
 }
@@ -175,8 +178,10 @@ async function runSingleShotCodingTask(chatFn: ChatFn, model: string, task: Codi
     ]);
     const raw = response.content.trim();
     const fenced = raw.match(/```(?:[a-z]*)\n([\s\S]*?)```/);
-    const code = fenced ? fenced[1].trim() : raw;
-    const target = resolveCodingPath(root, task.allowedFiles[0]);
+    const code = fenced ? (fenced[1] ?? "").trim() : raw;
+    const firstAllowed = task.allowedFiles[0];
+    if (!firstAllowed) throw new Error(`coding task ${task.id} declares no allowed file`);
+    const target = resolveCodingPath(root, firstAllowed);
     fs.mkdirSync(path.dirname(target), { recursive: true });
     fs.writeFileSync(target, code + "\n");
     const publicResult = runCodingVerifier(task, root, "public");

package/extensions/opl-simplebench/index.ts

@@ -1,7 +1,9 @@
 import type { ExtensionAPI, AgentToolResult } from "@earendil-works/pi-coding-agent";
+import { existsSync, unlinkSync } from "node:fs";
+import { Type } from "typebox";
 import { debugLog } from "./util/debug";
 import { detectProvider } from "./util/providers";
-import { readTestConfig, TOOL_SUPPORT_CACHE_PATH, type ModelTestUserConfig, type RunSequenceProfile } from "./util/config";
+import { readTestConfig, toolSupportCachePath, type ModelTestUserConfig, type RunSequenceProfile } from "./util/config";
 import type { SimplebenchOptions } from "./types";
 import { createBenchmark } from "./benchmark";
 
@@ -107,9 +109,8 @@ pi.registerCommand("simplebench", {
 
     if (arg === "--clear-cache") {
       try {
-        const fs = require("node:fs");
-        if (fs.existsSync(TOOL_SUPPORT_CACHE_PATH)) {
-          fs.unlinkSync(TOOL_SUPPORT_CACHE_PATH);
+        if (existsSync(toolSupportCachePath())) {
+          unlinkSync(toolSupportCachePath());
           ctx.ui.notify("Tool support cache cleared successfully", "info");
         } else {
           ctx.ui.notify("No cache file found to clear", "info");
@@ -141,7 +142,9 @@ pi.registerCommand("simplebench", {
           ctx.ui.notify(`Cooling down ${resolved.pauseMs / 1000}s before iteration ${i + 1}/${resolved.runs.length}...`, "info");
           await new Promise(resolve => setTimeout(resolve, resolved.pauseMs));
         }
-        const { entry, options } = resolved.runs[i];
+        const run = resolved.runs[i];
+        if (!run) continue;
+        const { entry, options } = run;
         const model = options.model || sequenceModel;
         ctx.ui.notify(`Iteration ${i + 1}/${resolved.runs.length} (${entry}) on ${model}...`, "info");
         try {
@@ -236,20 +239,17 @@ pi.registerTool({
   promptGuidelines: [
     "When the user asks to test or evaluate a model, call simplebench with the model name.",
   ],
-  parameters: {
-    type: "object",
-    properties: {
-      model: { type: "string", description: "Model name to test. If omitted, tests the current model." },
-      no_artifact: { type: "boolean", description: "If true, do not write the detailed JSON audit artifact to the current working directory." },
-      thinking_max: { type: "boolean", description: "Request maximum reasoning on an OpenAI-compatible provider or a direct Bedrock model that advertises max thinking. Omit to use provider defaults." },
-      coding_lite: { type: "boolean", description: "Run only the six execution-backed coding tasks in disposable directories." },
-      test_all: { type: "boolean", description: "Run the existing baseline, coding-lite, and deterministic grounded research tests." },
-      research_live: { type: "boolean", description: "Run configured live-search research as an integration smoke test; it does not affect recommendation." },
-      llama_server: { type: "boolean", description: "Capture /props and /metrics from configured llamaServerUrl. Inference routing is unchanged." },
-      llamagputop: { type: "boolean", description: "Capture configured llamagputopUrl /stats. The declared endpoint is authoritative; no Pi model match is required." },
-      tag: { type: "string", description: "Optional single-word label (letters, digits, dot, dash, underscore). Stored under benchmark.tag and prefixed onto the artifact file or bundle name." },
-    },
-  } as any,
+  parameters: Type.Object({
+    model: Type.Optional(Type.String({ description: "Model name to test. If omitted, tests the current model." })),
+    no_artifact: Type.Optional(Type.Boolean({ description: "If true, do not write the detailed JSON audit artifact to the session directory." })),
+    thinking_max: Type.Optional(Type.Boolean({ description: "Request maximum reasoning on an OpenAI-compatible provider or a direct Bedrock model that advertises max thinking. Omit to use provider defaults." })),
+    coding_lite: Type.Optional(Type.Boolean({ description: "Run only the six execution-backed coding tasks in disposable directories." })),
+    test_all: Type.Optional(Type.Boolean({ description: "Run the existing baseline, coding-lite, and deterministic grounded research tests." })),
+    research_live: Type.Optional(Type.Boolean({ description: "Run configured live-search research as an integration smoke test; it does not affect recommendation." })),
+    llama_server: Type.Optional(Type.Boolean({ description: "Capture /props and /metrics from configured llamaServerUrl. Inference routing is unchanged." })),
+    llamagputop: Type.Optional(Type.Boolean({ description: "Capture configured llamagputopUrl /stats. The declared endpoint is authoritative; no Pi model match is required." })),
+    tag: Type.Optional(Type.String({ description: "Optional single-word label (letters, digits, dot, dash, underscore). Stored under benchmark.tag and prefixed onto the artifact file or bundle name." })),
+  }),
   execute: async (_toolCallId, _params, _signal, _onUpdate, ctx) => {
     const params = _params as any;
     if (params?.tag !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(String(params.tag))) {

package/extensions/opl-simplebench/llama-server.ts

@@ -115,14 +115,15 @@ export function parsePrometheusMetrics(text: string): Record<string, number> {
     if (!match) continue;
     const value = Number(match[2]);
     // llama.cpp namespaces every metric as `llamacpp:<name>`; key on the bare name.
-    if (Number.isFinite(value)) out[match[1].replace(/^llamacpp:/, "")] = value;
+    const name = match[1];
+    if (name && Number.isFinite(value)) out[name.replace(/^llamacpp:/, "")] = value;
   }
   return out;
 }
 
 export function diffPrometheusMetrics(before: Record<string, number>, after: Record<string, number>): Record<string, number> {
   const out: Record<string, number> = {};
-  for (const key of Object.keys(after)) out[key] = after[key] - (before[key] ?? 0);
+  for (const key of Object.keys(after)) out[key] = (after[key] ?? 0) - (before[key] ?? 0);
   return out;
 }
 

package/extensions/opl-simplebench/report.ts

@@ -10,12 +10,6 @@ export function formatInstructionScore(instructions: { pass: boolean; score: str
 export function formatTestSummary(tests: Array<{ name: string; pass: boolean; score: string }>, totalMs: number): string[] {
   return [section("SUMMARY"), ...tests.map(t => t.pass ? ok(`${t.name}: ${t.score}`) : fail(`${t.name}: ${t.score}`)), info(`Total time: ${msHuman(totalMs)}`), info(`Score: ${tests.filter(t => t.pass).length}/${tests.length} tests passed`)];
 }
-/** Legacy-compatible category recommendation. New runner uses recommendation() above. */
-export function formatRecommendation(model: string, passed: number, total: number): string[] {
-  const label = passed === total ? "STRONG" : passed >= total - 1 ? "GOOD" : passed >= total - 2 ? "USABLE" : "WEAK";
-  return [section("RECOMMENDATION"), label === "WEAK" ? fail(`${model} is ${label}`) : ok(`${model} is ${label}`)];
-}
-
 export function recommendation(reasoning: string, instructionsPass: boolean, toolsPass: boolean, coding?: { passed: number; total: number; efficiency?: { strong: number; moderate: number; weak: number; fail: number } }) {
   const reasoningPass = reasoning === "STRONG" || reasoning === "MODERATE";
   // Efficiency-weighted coding pass: STRONG=1.0, MODERATE=0.7, WEAK=0.4, FAIL=0

package/extensions/opl-simplebench/research.ts

@@ -95,15 +95,16 @@ export function verifyGroundedResearch(markdown: string, fixture: GroundedResear
   const findings = markdown.match(/##\s+Findings\s*\n([\s\S]*?)(?=\n##\s|$)/i)?.[1] ?? "";
   const sourceSection = markdown.match(/##\s+Sources\s*\n([\s\S]*?)(?=\n##\s|$)/i)?.[1] ?? "";
   const citedClaims = [...findings.matchAll(/^\s*[-*]\s+(.+?)\s+\[([A-Za-z0-9]+)\]\s*$/gm)];
-  const unknownCitations = citedClaims.map(match => match[2]).filter(id => !knownSources.has(id));
+  const citationIds = citedClaims.map((match) => match[2] ?? "");
+  const unknownCitations = citationIds.filter((id) => !knownSources.has(id));
   const missingClaims: string[] = [];
   const wrongCitations: string[] = [];
   for (const claim of fixture.claims) {
-    const match = citedClaims.find(item => normalizeClaim(item[1]) === normalizeClaim(claim.text));
+    const match = citedClaims.find((item) => normalizeClaim(item[1] ?? "") === normalizeClaim(claim.text));
     if (!match) missingClaims.push(claim.text);
-    else if (match[2] !== claim.sourceId) wrongCitations.push(claim.text);
+    else if ((match[2] ?? "") !== claim.sourceId) wrongCitations.push(claim.text);
   }
-  for (const id of new Set(citedClaims.map(match => match[2]).filter(id => knownSources.has(id)))) {
+  for (const id of new Set(citationIds.filter((id) => knownSources.has(id)))) {
     const source = knownSources.get(id)!;
     if (!new RegExp(`\\[${id}\\]\\(${source.url.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\)`).test(sourceSection)) wrongCitations.push(id);
   }

package/extensions/opl-simplebench/scoring.ts

@@ -6,7 +6,7 @@ const NUMERIC_ANSWER = /^\d+$/;
 
 function lastNumericToken(line: string): string | null {
   const tokens = line.match(/\d+/g);
-  return tokens && tokens.length ? tokens[tokens.length - 1] : null;
+  return tokens?.[tokens.length - 1] ?? null;
 }
 
 export function extractAnswer(response: string, expected?: string): { answer: string; method: string } {
@@ -16,7 +16,9 @@ export function extractAnswer(response: string, expected?: string): { answer: st
   // bare-number remnants that thinking templates leak after the real final line.
   if (expected && NUMERIC_ANSWER.test(expected)) {
     for (let i = lines.length - 1; i >= 0; i -= 1) {
-      const token = lastNumericToken(lines[i]);
+      const line = lines[i];
+      if (line === undefined) continue;
+      const token = lastNumericToken(line);
       if (token === expected) {
         return { answer: token, method: i === lines.length - 1 ? "final-line" : "expected-last-numeric" };
       }

package/extensions/opl-simplebench/util/config.ts

@@ -8,8 +8,8 @@
  */
 
 import * as fs from "node:fs";
-import * as os from "node:os";
 import * as path from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
 
 // ============================================================================
 // Configuration Constants
@@ -59,9 +59,15 @@ export const CONFIG = {
 // User Configuration Overrides
 // ============================================================================
 
-const TEST_CONFIG_DIR = process.env.PI_AGENT_DIR || path.join(os.homedir(), ".pi", "agent");
-export const TEST_CONFIG_PATH = path.join(TEST_CONFIG_DIR, "configs", "opl-simplebench.json");
-const LEGACY_TEST_CONFIG_PATH = path.join(TEST_CONFIG_DIR, "simplebench-config.json");
+/** Resolved per call so a custom `PI_AGENT_DIR` (pi's own agent dir) is honoured. */
+export function configPath(): string {
+  return path.join(getAgentDir(), "configs", "opl-simplebench.json");
+}
+
+/** Pre-rename location, still read when the current path is absent. */
+function legacyConfigPath(): string {
+  return path.join(getAgentDir(), "simplebench-config.json");
+}
 
 /** Shape of the user configuration file. */
 export interface ModelTestUserConfig {
@@ -107,8 +113,8 @@ export interface RunSequenceProfile {
  */
 export function readTestConfig(): ModelTestUserConfig {
   try {
-    const configPath = fs.existsSync(TEST_CONFIG_PATH) ? TEST_CONFIG_PATH : LEGACY_TEST_CONFIG_PATH;
-    if (fs.existsSync(configPath)) return JSON.parse(fs.readFileSync(configPath, "utf-8")) as ModelTestUserConfig;
+    const file = fs.existsSync(configPath()) ? configPath() : legacyConfigPath();
+    if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, "utf-8")) as ModelTestUserConfig;
   } catch { /* config read/parse failure is non-critical — defaults are used */ }
   return {};
 }
@@ -161,8 +167,9 @@ export const WEATHER_TOOL_DEFINITION = {
 // Tool Support Cache (path only — backing file is no longer written)
 // ============================================================================
 
-const TOOL_SUPPORT_CACHE_DIR = path.join(os.homedir(), ".pi", "agent", "cache");
-export const TOOL_SUPPORT_CACHE_PATH = path.join(TOOL_SUPPORT_CACHE_DIR, "tool_support.json");
+export function toolSupportCachePath(): string {
+  return path.join(getAgentDir(), "cache", "tool_support.json");
+}
 
 // ============================================================================
 // ChatFn Abstraction

package/extensions/opl-simplebench/util/exec-env.ts

@@ -0,0 +1,36 @@
+/**
+ * Environment for processes that run model-authored code.
+ *
+ * `coding_lite` and `test_all` write the model's answer to disk and execute it,
+ * so whatever the model wrote runs with this process's parent environment. The
+ * harness is normally started from a shell that holds provider keys, cloud
+ * credentials and agent paths, and none of those belong in front of generated
+ * code. This is therefore an allowlist, not a denylist: anything not listed here
+ * is absent from the child, including every `*_API_KEY`, `AWS_*`, `GH_*` and
+ * `PI_*` variable.
+ *
+ * `NODE_OPTIONS` and `NODE_PATH` are deliberately never forwarded: both can make
+ * Node load attacker-chosen files.
+ */
+export const INHERITED_KEYS = [
+  // POSIX: node resolution, temp files, locale and timezone.
+  "PATH", "HOME", "TMPDIR", "TEMP", "TMP", "LANG", "LC_ALL", "TZ", "SHELL", "USER", "LOGNAME",
+  // TLS trust the child may legitimately need.
+  "SSL_CERT_FILE", "SSL_CERT_DIR", "NODE_EXTRA_CA_CERTS",
+  // Terminal capabilities only; no secrets live here.
+  "TERM", "NO_COLOR", "FORCE_COLOR", "COLORTERM",
+  // Windows: spawning and node resolution fail without these.
+  "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT", "USERPROFILE", "HOMEDRIVE", "HOMEPATH",
+  "APPDATA", "LOCALAPPDATA",
+];
+
+/** Build the environment for a child that executes untrusted, model-authored code. */
+export function scrubbedEnv(base: NodeJS.ProcessEnv = process.env): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const key of INHERITED_KEYS) {
+    const value = base[key];
+    // NUL-containing values are rejected by spawn(); drop them instead of throwing.
+    if (typeof value === "string" && value.length > 0 && !value.includes("\0")) out[key] = value;
+  }
+  return out;
+}

package/extensions/opl-simplebench/util/format.ts

@@ -258,7 +258,7 @@ export function sanitizeForReport(s: string, maxLines = 40): string {
   // Use strict detection: require a closing tag AND a common HTML opening tag to avoid
   // false positives on code that mentions HTML-like syntax (e.g., "use <Item> from 'react'")
   if (/<!DOCTYPE\b|<html[\s>]/i.test(cleaned) || (/<[a-z][\s\S]*>/i.test(cleaned) && cleaned.includes("</") && /<(?:div|span|p|head|body|html|table|form|script)\b/i.test(cleaned))) {
-    const firstLine = cleaned.split("\n")[0];
+    const firstLine = cleaned.split("\n")[0] ?? "";
     return truncate(firstLine, 200) + "\n  ℹ️  (HTML response truncated)";
   }
 
@@ -365,7 +365,7 @@ function parseParamCount(s: string): number | undefined {
   const match = str.match(/^([\d.]+)\s*([bmt]?|a(?:pple)?)$/);
   if (!match) return undefined;
 
-  const num = parseFloat(match[1]);
+  const num = parseFloat(match[1] ?? "");
   if (isNaN(num) || num <= 0) return undefined;
 
   const suffix = match[2];