@openlines/opl-pi-sht 0.1.21 → 0.2.0

package/extensions/opl-modes/utils.ts

@@ -1,16 +1,16 @@
-/** Pure utilities: isSafeCommand, extractPlanText, plan file I/O, color helpers */
+/** Pure utilities: read-only Bash gate, plan file I/O, color helpers */
 
 import type { Theme, ThemeColor } from "@earendil-works/pi-coding-agent";
 import { existsSync, readFileSync, readdirSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 
-import { SAFE_COMMAND_PATTERNS, DESTRUCTIVE_PATTERNS, PLAN_DIR, PLAN_FILE_PREFIX } from "./config.js";
+import { PLAN_DIR, PLAN_FILE_PREFIX } from "./config.js";
 import type { PlanFileSummary } from "./types.js";
 
 /** Serialize async model changes so the final requested model wins. */
 export function createLatestModelQueue() {
   let latest = 0;
-  let tail = Promise.resolve();
+  let tail: Promise<unknown> = Promise.resolve();
 
   return <T>(change: () => Promise<T>): Promise<T | undefined> => {
     const request = ++latest;
@@ -19,17 +19,124 @@ export function createLatestModelQueue() {
   };
 }
 
-/** Check if command matches safe patterns and not destructive patterns. */
-export function isSafeCommand(command: string): boolean {
-  return SAFE_COMMAND_PATTERNS.some((p) => p.test(command))
-    && !isDestructive(command, DESTRUCTIVE_PATTERNS);
+const stripQuotes = (text: string): string => text.replace(/["'\\]/g, "");
+
+/**
+ * Blind split on every operator, quotes ignored — used only for the blocklist scan, where
+ * over-splitting is fail-safe. A lone `&` counts as a separator unless it belongs to `&&`,
+ * `||`, or an `>&`/`<&` descriptor duplication (`echo hi 2>&1` stays one command).
+ */
+const BLIND_SPLIT = /\$\(|`|[<>]\(|&&|\|\||(?<![<>&])&(?![>&])|[;|\n\r]/;
+
+/** Quote/substitution nesting level. `quote` is the string quote open inside this level. */
+interface Frame { kind: "root" | "sub" | "tick"; quote: "" | "'" | '"'; parens: number }
+
+/**
+ * Split a command into the pieces the shell actually executes, so each one can be required
+ * to clear the allowlist: chaining (`&&`, `||`), backgrounding (`&`), pipes, `;`, newlines,
+ * process substitution, and the payload of `$(...)`/backticks. Quote-aware, so `grep "a|b"`
+ * stays one command, while a substitution inside double quotes still expands —
+ * `echo "$(node -e ...)"` yields the `node` payload as its own segment instead of blanking
+ * the whole string away. Quote characters are dropped from the segments (matching is
+ * prefix-anchored, so a leading `"` must not defeat a safe pattern).
+ */
+function commandSegments(command: string): string[] {
+  const out: string[] = [];
+  const stack: Frame[] = [{ kind: "root", quote: "", parens: 0 }];
+  let buf = "";
+  const flush = () => {
+    const s = buf.trim();
+    if (s) out.push(s);
+    buf = "";
+  };
+  for (let i = 0; i < command.length; i++) {
+    const ch = command[i];
+    const top = stack[stack.length - 1];
+    // Single quotes are literal: nothing expands, only the closing ' matters.
+    if (top.quote === "'") {
+      if (ch === "'") top.quote = "";
+      else buf += ch;
+      continue;
+    }
+    // Double quotes keep their text literal, except for `$(...)` and backticks, which run.
+    if (top.quote === '"') {
+      if (ch === '"') { top.quote = ""; continue; }
+      if (ch === "\\") { i++; continue; } // \X is a literal X, so `\$(x)` does not expand
+      if (ch === "$" && command[i + 1] === "(") { stack.push({ kind: "sub", quote: "", parens: 0 }); flush(); i++; continue; }
+      if (ch === "`") { stack.push({ kind: "tick", quote: "", parens: 0 }); flush(); continue; }
+      buf += ch;
+      continue;
+    }
+    if (ch === "'") { top.quote = "'"; continue; }
+    if (ch === '"') { top.quote = '"'; continue; }
+    if (ch === "\\") { buf += ch + (command[++i] ?? ""); continue; }
+    if (ch === "$" && command[i + 1] === "(") { stack.push({ kind: "sub", quote: "", parens: 0 }); flush(); i++; continue; }
+    if (ch === "`") {
+      if (top.kind === "tick") stack.pop();
+      else stack.push({ kind: "tick", quote: "", parens: 0 });
+      flush();
+      continue;
+    }
+    if (top.kind === "sub") {
+      if (ch === "(") top.parens++;
+      else if (ch === ")") {
+        if (top.parens === 0) { stack.pop(); flush(); continue; }
+        top.parens--;
+      }
+    } else if ((ch === "<" || ch === ">") && command[i + 1] === "(") {
+      stack.push({ kind: "sub", quote: "", parens: 0 });
+      flush();
+      i++;
+      continue;
+    }
+    if (ch === ";" || ch === "|" || ch === "\n" || ch === "\r") { flush(); continue; }
+    if (ch === "&") {
+      if (command[i + 1] === "&") { i++; flush(); continue; } // &&
+      // A lone & backgrounds the command before it. `>&`/`<&` is descriptor duplication and
+      // `&>` is a redirect, so neither starts a new command here (the blocklist judges those).
+      if (!/[<>&|]/.test(command[i - 1] ?? "") && command[i + 1] !== ">") { flush(); continue; }
+    }
+    buf += ch;
+  }
+  flush();
+  return out;
+}
+
+/** Quote-blind segments for the blocklist, on the raw text and its quote-stripped skeleton. */
+function blindSegments(text: string): string[] {
+  return text.split(BLIND_SPLIT).map((s) => s.trim()).filter((s) => s.length > 0);
 }
 
-/** Whether a command, or its quote/backslash-stripped skeleton, matches any destructive
- *  pattern. Stripping defeats shell expands like `r"m"` -> `rm` that dodge \brm\b. */
+/** Whether a destructive pattern matches the command, any of its segments, or either
+ *  with quotes/backslashes stripped, so `r"m"` and `-del"ete"` cannot dodge the list. */
 export function isDestructive(command: string, patterns: RegExp[]): boolean {
-  const skeleton = command.replace(/["'\\]/g, "");
-  return patterns.some((p) => p.test(command) || p.test(skeleton));
+  const haystacks = [
+    command,
+    stripQuotes(command),
+    ...blindSegments(command).flatMap((s) => [s, stripQuotes(s)]),
+  ];
+  return patterns.some((p) => haystacks.some((h) => p.test(h)));
+}
+
+/**
+ * Why a Bash command must be blocked in a read-only mode, or null when it is allowed.
+ * Every shell segment has to match a safe pattern (so `cat f && node -e '...'` cannot
+ * ride the first command's allowance), and no destructive pattern may match anywhere.
+ * An undefined or empty safe list means "no allowlist gate".
+ */
+export function bashBlockReason(
+  command: string,
+  safePatterns?: RegExp[],
+  destructivePatterns?: RegExp[],
+): string | null {
+  if (safePatterns && safePatterns.length > 0) {
+    const offender = commandSegments(command).find((s) => !safePatterns.some((p) => p.test(s)));
+    if (offender !== undefined) return `not in safe pattern list: ${offender}`;
+  }
+  if (destructivePatterns && destructivePatterns.length > 0 && isDestructive(command, destructivePatterns)) {
+    return `destructive pattern in: ${command}`;
+  }
+  return null;
 }
 
 const PLAN_ACTION_VERB = /^\s*\d+\.\s+(?:\*{1,2})?(?:add|creat|updat|fix|remov|refactor|implement|modif|chang|edit|writ|build|run|install|configur|set\s+up|delet|mov|renam|inject|migrat|replac|extract|test|deploy|integrat|convert)/i;
@@ -90,6 +197,8 @@ export function sanitizePlanName(name: string): string | null {
   if (!trimmed) return null;
   if (trimmed.includes("/") || trimmed.includes("\\") || trimmed.includes("..")) return null;
   if (!/^[\w\s.-]+$/.test(trimmed)) return null;
+  // A name with no letter or digit (".", "--", ". ") would create plan-.md and an empty title.
+  if (!/\w/.test(trimmed)) return null;
   return trimmed.replace(/\s+/g, "-");
 }
 
@@ -134,7 +243,9 @@ export function isHexColor(color: string): boolean {
 
 /** Convert hex color string to ANSI truecolor escape. */
 function hexToAnsi(hex: string): string {
-  const h = hex.replace("#", "");
+  // Accept the #abc shorthand so a three-digit value renders the same as #aabbcc.
+  let h = hex.replace("#", "");
+  if (/^[0-9a-fA-F]{3}$/.test(h)) h = h.split("").map((c) => c + c).join("");
   if (!/^[0-9a-fA-F]{6}$/.test(h)) return "";
   const r = parseInt(h.slice(0, 2), 16);
   const g = parseInt(h.slice(2, 4), 16);

package/extensions/opl-simplebench/benchmark.ts

@@ -312,7 +312,9 @@ function makePiBedrockThinkingChatFn(modelConfig: BenchmarkModel): ChatFn {
       ...(systemPrompt ? { systemPrompt } : {}),
       messages: messages.filter(m => m.role !== "system") as any,
       ...(tools?.length ? { tools } : {}),
-    }, { reasoning: "max" });
+      // pi-ai's TranscriptContext carries a private brand, so a literal cannot satisfy it
+      // structurally; the fields above are the shape streamSimple actually reads.
+    } as any, { reasoning: "max" });
     const message: any = await stream.result();
     const content = (message.content || []).filter((block: any) => block.type === "text").map((block: any) => block.text).join("");
     const toolCalls = (message.content || []).filter((block: any) => block.type === "toolCall").map((block: any) => ({
@@ -387,13 +389,22 @@ function makeOllamaToolChatFn(): ChatFn {
 
 // ── Ollama Chat Functions ─────────────────────────────────────────────
 
+/** Both Ollama chat helpers return this shape; the unused fields stay undefined per mode. */
+interface OllamaChatResult {
+  response: any;
+  elapsedMs: number;
+  requestCount?: number;
+  retryCount?: number;
+  timeToFirstTokenMs?: number | null;
+}
+
 async function ollamaChat(
   model: string,
   messages: Array<{ role: string; content: string }>,
   options: Record<string, unknown> = {},
   timeoutMs = CONFIG.DEFAULT_TIMEOUT_MS,
   retries = CONFIG.MAX_RETRIES
-): Promise<{ response: any; elapsedMs: number }> {
+): Promise<OllamaChatResult> {
   const { tools, ...generationOptions } = options as any;
   const body: any = { model, messages, stream: false, ...(tools ? { tools } : {}), ...(Object.keys(generationOptions).length ? { options: generationOptions } : {}) };
   const url = `${ollamaBase()}/api/chat`;
@@ -425,7 +436,9 @@ async function ollamaChat(
         throw new Error(`Empty response from Ollama after ${attempt + 1} attempt(s)`);
       }
       const parsed = JSON.parse(text);
-      return { response: parsed, elapsedMs };
+      // Surface the retry loop so artifacts record real request/retry counts (metricsFromChat
+      // used to receive undefined and silently report 1 request, 0 retries).
+      return { response: parsed, elapsedMs, requestCount: attempt + 1, retryCount: attempt };
     } catch (e: unknown) {
       const msg = e instanceof Error ? e.message : String(e);
       if (e instanceof Error && e.name === "AbortError") {
@@ -455,7 +468,7 @@ async function ollamaChatStream(
   messages: Array<{ role: string; content: string }>,
   options: Record<string, unknown> = {},
   timeoutMs = CONFIG.DEFAULT_TIMEOUT_MS,
-): Promise<{ response: any; elapsedMs: number }> {
+): Promise<OllamaChatResult> {
   const { tools, ...generationOptions } = options as any;
   const body: any = { model, messages, stream: true, ...(tools ? { tools } : {}), ...(Object.keys(generationOptions).length ? { options: generationOptions } : {}) };
   const url = `${ollamaBase()}/api/chat`;
@@ -484,6 +497,7 @@ async function ollamaChatStream(
     let thinkingContent = "";
     const toolCalls: any[] = [];
     let done = false;
+    let timeToFirstTokenMs: number | null = null;
 
     const reader = res.body.getReader();
     const decoder = new TextDecoder();
@@ -497,6 +511,9 @@ async function ollamaChatStream(
       for (const line of lines) {
         try {
           const parsed = JSON.parse(line);
+          if (timeToFirstTokenMs === null && (parsed.message?.content || parsed.message?.thinking || parsed.message?.tool_calls)) {
+            timeToFirstTokenMs = Date.now() - start;
+          }
           if (parsed.message?.content) messageContent += parsed.message.content;
           if (parsed.message?.thinking) thinkingContent += parsed.message.thinking;
           if (parsed.message?.tool_calls) toolCalls.push(...parsed.message.tool_calls);
@@ -521,7 +538,7 @@ async function ollamaChatStream(
       done: true,
     };
 
-    return { response, elapsedMs };
+    return { response, elapsedMs, timeToFirstTokenMs };
   } catch (e: unknown) {
     if (e instanceof Error && e.name === "AbortError") {
       throw new Error(`Ollama API timed out after ${msHuman(timeoutMs)}`);
@@ -562,7 +579,7 @@ async function testReasoningExtended(chatFn: ChatFn, model: string, onProgress?:
       const msg = result.content.trim();
       const scored = scoreReasoning(msg, test.expectedAnswer);
       const answer = scored.answer;
-      results.push({ name: test.name, category: test.category, prompt: test.prompt, response: msg, elapsedMs: result.elapsedMs, metrics: metricsFromChat({ ...result, startedAt: result.startedAt ?? requestedAt, finishedAt: result.finishedAt ?? new Date().toISOString() }), score: scored.score, answer, expectedAnswer: test.expectedAnswer, pass: scored.pass, details: scored.details });
+      results.push({ name: test.name, category: test.category, prompt: test.prompt, response: msg, elapsedMs: result.elapsedMs, metrics: metricsFromChat({ ...result, startedAt: result.startedAt ?? requestedAt, finishedAt: result.finishedAt ?? new Date().toISOString() }), score: scored.score, answer, expectedAnswer: test.expectedAnswer, pass: scored.pass });
       onProgress?.(`[1/3] Reasoning ${i + 1}/${total}: ${test.name} → ${scored.score}`);
     } catch (e: any) {
       results.push({ name: test.name, category: test.category, prompt: test.prompt, response: null, error: e?.message || String(e), elapsedMs: 0, metrics: emptyMetrics(), score: "ERROR", answer: "?", expectedAnswer: test.expectedAnswer, pass: false });
@@ -683,7 +700,7 @@ async function testModelExtended(model: string, ctx?: any, options: SimplebenchO
   const providerInfo = ctx ? detectProvider(ctx) : { kind: "ollama" as const, name: "ollama" };
   const resolvedModel = resolveBenchmarkModel(ctx, model);
   const thinking = resolveThinkingMode(providerInfo, resolvedModel.model, options.thinkingMax === true);
-  const suite = options.testAll ? "test-all" : options.codingLite ? "coding-lite" : "baseline";
+  const suite: "baseline" | "coding-lite" | "test-all" = options.testAll ? "test-all" : options.codingLite ? "coding-lite" : "baseline";
 
   // Direct llama-server probes are separate from the provider route. This is
   // intentional: --llamagputop also works when inference goes through LiteLLM.

package/extensions/opl-simplebench/index.ts

@@ -65,11 +65,10 @@ export default function (pi: ExtensionAPI) {
 
 pi.registerCommand("simplebench", {
   description: "Benchmark a model with auditable closed-answer, instruction, and tool-use tests.",
-  detailedHelp: "\n\n🔍 Simplebench Extension\n\nThis extension tests AI models across multiple dimensions:\n• Closed-answer contract: 20 deterministic final-line answers (logic, math, spatial, commonsense)\n• Tool Usage: Ability to use available tools effectively\n• Instruction Following: How well the model follows complex JSON instructions\n• Coding Lite: Six isolated, execution-backed coding tasks\n\n📋 Usage Examples:\n  /simplebench                    # Test current model\n  /simplebench <model>           # Test a specific model\n  /simplebench --all             # Test all Ollama models\n  /simplebench <model> --3ptest  # Run the default 3ptest baseline explicitly\n  /simplebench <model> --coding-lite # Run only coding tasks\n  /simplebench <model> --test-all # Run baseline, coding, and grounded research\n  /simplebench <model> --research-live # Add live search integration smoke test\n  /simplebench --all --test-all  # Run complete suite for every Ollama model\n  /simplebench <model> --sequence[=<name>] # Run a configured runSequence profile\n  /simplebench <model> --thinking-max # Request max reasoning\n  /simplebench <model> --llama-server # Capture configured /props and /metrics\n  /simplebench <model> --llamagputop # Capture configured llama.cpp stats\n  /simplebench <model> --tag=coldrun # Label the run: benchmark.tag plus artifact filename prefix\n  /simplebench --help            # Show this help\n  /simplebench --clear-cache     # Clear tool support cache\n\nCoding tasks run in disposable directories and never access the user repository.\n",
   getArgumentCompletions: async (prefix) => {
     try {
       const models = await getOllamaModels();
-      return models.map(m => ({ label: m, description: `Test ${m}` }))
+      return models.map(m => ({ value: m, label: m, description: `Test ${m}` }))
         .filter(m => m.label.startsWith(prefix));
     } catch (err) { debugLog("simplebench", "failed to get model completions", err); return []; }
   },
@@ -151,7 +150,7 @@ pi.registerCommand("simplebench", {
           pi.sendMessage({
             customType: "simplebench-report",
             content: report,
-            display: { type: "content", content: report },
+            display: true,
             details: { model, timestamp: new Date().toISOString() },
           });
         } catch (e: any) {
@@ -191,7 +190,7 @@ pi.registerCommand("simplebench", {
           pi.sendMessage({
             customType: "simplebench-report",
             content: report,
-            display: { type: "content", content: report },
+            display: true,
             details: { model, timestamp: new Date().toISOString() },
           });
         } catch (e: any) {
@@ -214,7 +213,7 @@ pi.registerCommand("simplebench", {
       pi.sendMessage({
         customType: "simplebench-report",
         content: report,
-        display: { type: "content", content: report },
+        display: true,
         details: { model, timestamp: new Date().toISOString() },
       });
     } catch (e: any) {
@@ -256,6 +255,7 @@ pi.registerTool({
     if (params?.tag !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(String(params.tag))) {
       return {
         content: [{ type: "text", text: "--tag must be a single word (letters, digits, dot, dash, underscore)" }],
+        details: undefined,
         isError: true,
       } as AgentToolResult;
     }
@@ -264,6 +264,7 @@ pi.registerTool({
     if (!model) {
       return {
         content: [{ type: "text", text: "No model currently selected to test." }],
+        details: undefined,
         isError: true,
       } as AgentToolResult;
     }
@@ -271,6 +272,7 @@ pi.registerTool({
       const report = await testModel(model, ctx, options);
       return {
         content: [{ type: "text", text: report }],
+        details: undefined,
         isError: false,
       } as AgentToolResult;
     } catch (e: any) {
@@ -281,6 +283,7 @@ pi.registerTool({
       
       return {
         content: [{ type: "text", text: errorMessage }],
+        details: undefined,
         isError: true,
       } as AgentToolResult;
     }

package/extensions/opl-simplebench/research.ts

@@ -88,7 +88,10 @@ function normalizeClaim(value: string): string {
 }
 
 export function verifyGroundedResearch(markdown: string, fixture: GroundedResearchFixture): { passed: boolean; missingClaims: string[]; wrongCitations: string[]; unknownCitations: string[] } {
-  const knownSources = new Map(fixture.sources.map(source => [source.id, source]));
+  // Key is string: citation ids arrive from a regex capture, so a "S1"|"S2"|"S3" key would reject them.
+  const knownSources = new Map<string, GroundedResearchFixture["sources"][number]>(
+    fixture.sources.map(source => [source.id, source]),
+  );
   const findings = markdown.match(/##\s+Findings\s*\n([\s\S]*?)(?=\n##\s|$)/i)?.[1] ?? "";
   const sourceSection = markdown.match(/##\s+Sources\s*\n([\s\S]*?)(?=\n##\s|$)/i)?.[1] ?? "";
   const citedClaims = [...findings.matchAll(/^\s*[-*]\s+(.+?)\s+\[([A-Za-z0-9]+)\]\s*$/gm)];

package/extensions/opl-simplebench/util/providers.ts

@@ -9,7 +9,12 @@ import * as fs from "node:fs";
 import * as path from "node:path";
 import os from "node:os";
 import { debugLog } from "./debug";
-import type { PiExtensionContext } from "../../../shared/types";
+
+/** Minimal shape of Pi's extension context that provider detection reads. Keeps this module
+ *  standalone instead of importing a Pi type the extension host resolves at load time. */
+interface PiExtensionContext {
+  model?: { provider?: string; id?: string };
+}
 
 // ============================================================================
 // Constants

package/extensions/opl-todo/index.ts

@@ -401,8 +401,8 @@ export default function (pi: ExtensionAPI) {
 			}
 			// Fallback: switch above must never fall through. Returning undefined here
 			// makes pi wrap it in a MouseRegion with an undefined child and crashes the TUI.
-			const fb = result.content[0];
-			return new Text(theme.fg("error", String(fb?.type === "text" ? fb.text : "Unknown todo result")), 0, 0);
+			// The action is known here, so report it instead of digging through content blocks.
+			return new Text(theme.fg("error", `Unhandled todo result for action "${String(details.action ?? "")}"`), 0, 0);
 		},
 	});
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openlines/opl-pi-sht",
-  "version": "0.1.21",
+  "version": "0.2.0",
   "description": "opl-pi-sht (shouldHaveThat) Pi coding-agent extension kit: token-efficient context, safe modes, fingerprinted /init, model benchmarking, web access, and browser automation.",
   "repository": {
     "type": "git",
@@ -45,7 +45,7 @@
     "test:opl-footer": "bun test tests/opl-footer-helpers.test.mjs && OPL_EXTENSION=opl-footer bun test tests/extension-smoke.test.mjs",
     "test:opl-init": "bun test tests/opl-init-crawl.test.mjs && OPL_EXTENSION=opl-init bun test tests/extension-smoke.test.mjs",
     "test:opl-input": "bun test tests/opl-input-style.test.mjs && OPL_EXTENSION=opl-input bun test tests/extension-smoke.test.mjs",
-    "test:opl-modes": "bun test tests/opl-modes-helpers.test.mjs && OPL_EXTENSION=opl-modes bun test tests/extension-smoke.test.mjs",
+    "test:opl-modes": "bun test tests/opl-modes-helpers.test.mjs tests/opl-modes-lifecycle.test.mjs && OPL_EXTENSION=opl-modes bun test tests/extension-smoke.test.mjs",
     "test:opl-questionnaire": "OPL_EXTENSION=opl-questionnaire bun test tests/extension-smoke.test.mjs",
     "test:opl-todo": "bun test tests/opl-todo-config.test.mjs tests/opl-todo-render.test.mjs && OPL_EXTENSION=opl-todo bun test tests/extension-smoke.test.mjs",
     "test:opl-webaccess": "bun test tests/opl-webaccess-helpers.test.mjs && OPL_EXTENSION=opl-webaccess bun test tests/extension-smoke.test.mjs",