@openlines/opl-pi-sht 0.1.15 → 0.1.16

package/README.md

@@ -200,7 +200,7 @@ npx playwright install chromium
 
 A Pi package (npm or Git) still needs the one-time `npx playwright install chromium` command shown above.
 
-`opl-simplebench` writes a full JSON benchmark artifact in Pi's current working directory by default. `--test-all` additionally writes `research.md` and `page.html` beside `result.json` in a result bundle. Copy `configs/opl-simplebench.json.sample` to `~/.pi/agent/configs/opl-simplebench.json` to configure DDGS/SearXNG research and optional llama-server/llamagputop metadata endpoints. Use `/simplebench --no-artifact` or `simplebench({ no_artifact: true })` when responses must not be written to disk. Provider credentials remain outside tracked configuration; configure them through Pi provider settings, environment variables, or Pi authentication.
+`opl-simplebench` writes a full JSON benchmark artifact in Pi's current working directory by default. `--test-all` additionally writes `research.md` and `page.html` beside `result.json` in a result bundle. `/simplebench --sequence` runs a templated multi-run protocol (for example a model warm-up curve) defined in the `runSequence` block, with per-iteration tags, optional llama-server/llamagputop metrics, and a configurable pause between iterations. Copy `configs/opl-simplebench.json.sample` to `~/.pi/agent/configs/opl-simplebench.json` to configure DDGS/SearXNG research, optional llama-server/llamagputop metadata endpoints, and the run sequence. Use `/simplebench --no-artifact` or `simplebench({ no_artifact: true })` when responses must not be written to disk. Provider credentials remain outside tracked configuration; configure them through Pi provider settings, environment variables, or Pi authentication.
 
 ## Tests
 

package/configs/opl-simplebench.json.sample

@@ -1,8 +1,19 @@
 {
-  "_comment": "Copy to ~/.pi/agent/configs/opl-simplebench.json. Endpoints are optional and only used by their corresponding Simplebench flags.",
+  "_comment": "Copy to ~/.pi/agent/configs/opl-simplebench.json. Endpoints are optional and only used by their corresponding Simplebench flags. runSequence powers /simplebench --sequence: each entry is a flag string; llamaMetrics appends --llama-server --llamagputop to every entry; pauseMs sleeps between iterations.",
   "researchSearchProvider": "ddgs",
   "researchSearchUrl": "http://127.0.0.1:8000",
   "researchMaxResults": 5,
   "llamaServerUrl": "http://127.0.0.1:8080/v1",
-  "llamagputopUrl": "http://127.0.0.1:4321/stats"
+  "llamagputopUrl": "http://127.0.0.1:4321/stats",
+  "runSequence": {
+    "enabled": false,
+    "llamaMetrics": true,
+    "pauseMs": 5000,
+    "sequence": [
+      "--coding-lite --tag=coldest",
+      "--3ptest --tag=colder",
+      "--coding-lite --tag=semiwarm",
+      "--test-all --research-live --tag=warm"
+    ]
+  }
 }

package/extensions/opl-simplebench/README.md

@@ -8,12 +8,14 @@ Auditable Pi model benchmark for closed-answer contracts, JSON instruction follo
 /simplebench
 /simplebench <model>
 /simplebench <model> [--no-artifact] [--thinking-max]
+/simplebench <model> --3ptest
 /simplebench <model> --coding-lite
 /simplebench <model> --test-all
 /simplebench <model> --research-live
 /simplebench <model> --llama-server
 /simplebench <model> --llamagputop
 /simplebench <model> --tag=<word>
+/simplebench <model> --sequence
 /simplebench --all [--no-artifact]
 /simplebench --all --test-all
 /simplebench --help
@@ -39,6 +41,8 @@ simplebench({ model: "global.openai.gpt-5.6-terra", test_all: true })
 - Twenty fixed closed-answer contracts, one strict JSON instruction test, and one chained tool-call test.
 - Six opt-in execution-backed coding tasks in disposable directories (`--coding-lite`).
 - `--test-all` runs baseline, coding-lite, and deterministic grounded research; `--research-live` adds the configured live research-artifact integration smoke test. `--all` still selects every Ollama model.
+- `--3ptest` is the explicit form of the default baseline suite (reasoning, instruction following, tool calls). It changes nothing about the run; artifacts already name the baseline suite `3ptest`.
+- `--sequence` runs the templated multi-iteration benchmark stored in `opl-simplebench.json` (`runSequence`): sequential iterations, per-entry flags and `--tag`, optional `llamaMetrics` and `pauseMs` cooldown. See "Run a templated sequence" below.
 - `--llama-server` (`llama_server: true`) captures configured direct llama-server `/props` and `/metrics` without changing inference routing, so it works with a LiteLLM proxy.
 - `--llamagputop` (`llamagputop: true`) captures configured llama.cpp `/stats` metadata independently of the inference provider; its served model ID is authoritative.
 - `--tag=<word>` (`tag: "<word>"`) labels a run: stored as `benchmark.tag` and prefixed onto the artifact file or bundle name, for example `simplebench-coldrun-coding-lite-<model>-...`. The tag must be a single word (letters, digits, dot, dash, underscore).
@@ -80,6 +84,37 @@ Use this in sensitive environments. The terminal summary still reports category
 
 `--all` is intentionally limited to Ollama discovery. It writes one artifact per model unless `--no-artifact` is set.
 
+### Run a templated sequence
+
+```text
+/simplebench --sequence
+```
+
+Repeating a warm-up protocol (first-run-after-boot coding check, then baseline, then the full suite) means re-typing the same flags every session. `--sequence` reads the `runSequence` block from `~/.pi/agent/configs/opl-simplebench.json` and runs each entry as its own iteration against the current model (or the model passed with `--sequence`):
+
+```json
+{
+  "runSequence": {
+    "enabled": true,
+    "llamaMetrics": true,
+    "pauseMs": 5000,
+    "sequence": [
+      "--coding-lite --tag=coldest",
+      "--3ptest --tag=colder",
+      "--coding-lite --tag=semiwarm",
+      "--test-all --research-live --tag=warm"
+    ]
+  }
+}
+```
+
+- `enabled` gates the flag: `/simplebench --sequence` errors when it is missing or `false`. A bare `/simplebench` is unaffected.
+- Each `sequence` entry is the same flag string you would type after `/simplebench`, including its own `--tag`.
+- `llamaMetrics: true` appends `--llama-server --llamagputop` to every entry, so local server stats are captured per iteration without repeating the flags.
+- `pauseMs` sleeps between iterations (never after the last one), including after a failed iteration, so thermal/KV-cache cooldown stays consistent across the warm-up curve.
+- Iterations run sequentially and each writes its normal artifact/bundle; a failed iteration is reported and the sequence continues. A final summary line counts completed iterations.
+- Entries containing `--all` or `--sequence` are rejected before anything runs, as are invalid configurations (empty sequence, negative `pauseMs`) and invalid per-entry tags. The outer `--tag` is ignored; each entry carries its own.
+
 ### Grounded research in `--test-all`
 
 `--test-all` runs a deterministic agent workflow after coding-lite. `web_search` returns three benchmark-local source cards. The model must write every required finding as a Markdown bullet with its inline source ID (`[S1]`, `[S2]`) and list each cited source under `## Sources` as a Markdown link, for example `- [S1](https://research.fixture/S1)`. The verifier checks each exact claim-to-source relationship, source-list URL, tool trace, and the responsive minimalist `page.html` contract. It invokes no LLM judge, external search service, embedding model, or entailment model.
@@ -94,7 +129,7 @@ When coding-lite is enabled, each coding task is recorded as a `kind: "coding"`
 
 ### Configured metadata (`--llama-server`, `--llamagputop`)
 
-Copy `configs/opl-simplebench.json.sample` to `~/.pi/agent/configs/opl-simplebench.json`. It uses camelCase fields: `llamaServerUrl`, `llamagputopUrl`, `researchSearchProvider`, `researchSearchUrl`, and `researchMaxResults`. `--llama-server` captures `GET /props` and `/metrics` from configured `llamaServerUrl`; inference remains on the configured provider route, including LiteLLM. `--llamagputop` captures configured `llamagputopUrl` stats.
+Copy `configs/opl-simplebench.json.sample` to `~/.pi/agent/configs/opl-simplebench.json`. It uses camelCase fields: `llamaServerUrl`, `llamagputopUrl`, `researchSearchProvider`, `researchSearchUrl`, `researchMaxResults`, and `runSequence` (see "Run a templated sequence"). `--llama-server` captures `GET /props` and `/metrics` from configured `llamaServerUrl`; inference remains on the configured provider route, including LiteLLM. `--llamagputop` captures configured `llamagputopUrl` stats.
 
 `serverStats` is omitted entirely for runs without the flag. Its shape:
 

package/extensions/opl-simplebench/index.ts

@@ -1,7 +1,7 @@
 import type { ExtensionAPI, AgentToolResult } from "@earendil-works/pi-coding-agent";
 import { debugLog } from "./util/debug";
 import { detectProvider } from "./util/providers";
-import { TOOL_SUPPORT_CACHE_PATH } from "./util/config";
+import { readTestConfig, TOOL_SUPPORT_CACHE_PATH, type ModelTestUserConfig } from "./util/config";
 import type { SimplebenchOptions } from "./types";
 import { createBenchmark } from "./benchmark";
 
@@ -13,7 +13,25 @@ export function parseCommandArgs(args: string): SimplebenchOptions {
     tag = tagToken.slice("--tag=".length);
     if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(tag)) throw new Error(`--tag must be a single word (letters, digits, dot, dash, underscore): got "${tag}"`);
   }
-  return { model: tokens.find(token => !token.startsWith("--")), allModels: tokens.includes("--all"), writeArtifact: !tokens.includes("--no-artifact"), thinkingMax: tokens.includes("--thinking-max"), codingLite: tokens.includes("--coding-lite"), testAll: tokens.includes("--test-all"), researchLive: tokens.includes("--research-live"), llamaServer: tokens.includes("--llama-server"), llamagputop: tokens.includes("--llamagputop"), ...(tag ? { tag } : {}) };
+  return { model: tokens.find(token => !token.startsWith("--")), allModels: tokens.includes("--all"), writeArtifact: !tokens.includes("--no-artifact"), thinkingMax: tokens.includes("--thinking-max"), codingLite: tokens.includes("--coding-lite"), testAll: tokens.includes("--test-all"), researchLive: tokens.includes("--research-live"), llamaServer: tokens.includes("--llama-server"), llamagputop: tokens.includes("--llamagputop"), ...(tag ? { tag } : {}), ...(tokens.includes("--3ptest") ? { threePTest: true } : {}), ...(tokens.includes("--sequence") ? { sequence: true } : {}) };
+}
+
+export interface ResolvedRunSequence { runs: Array<{ entry: string; options: SimplebenchOptions }>; pauseMs: number }
+
+/** Expand the configured runSequence into per-iteration benchmark options. Throws on disabled/invalid configuration. */
+export function resolveRunSequence(userConfig: ModelTestUserConfig): ResolvedRunSequence {
+  const rs = userConfig.runSequence;
+  if (!rs?.enabled) throw new Error("runSequence is not enabled in opl-simplebench.json");
+  if (!Array.isArray(rs.sequence) || rs.sequence.length === 0) throw new Error("runSequence.sequence must be a non-empty array of flag strings");
+  const pauseMs = rs.pauseMs ?? 0;
+  if (typeof pauseMs !== "number" || !Number.isFinite(pauseMs) || pauseMs < 0) throw new Error(`runSequence.pauseMs must be a non-negative number: got ${JSON.stringify(rs.pauseMs)}`);
+  const runs = rs.sequence.map((entry: string) => {
+    const tokens = String(entry).trim().split(/\s+/);
+    if (tokens.includes("--all")) throw new Error(`runSequence entry cannot contain --all: "${entry}"`);
+    if (tokens.includes("--sequence")) throw new Error(`runSequence entry cannot contain --sequence: "${entry}"`);
+    return { entry: String(entry), options: parseCommandArgs(`${entry}${rs.llamaMetrics ? " --llama-server --llamagputop" : ""}`) };
+  });
+  return { runs, pauseMs };
 }
 
 export default function (pi: ExtensionAPI) {
@@ -22,7 +40,7 @@ export default function (pi: ExtensionAPI) {
 
 pi.registerCommand("simplebench", {
   description: "Benchmark a model with auditable closed-answer, instruction, and tool-use tests.",
-  detailedHelp: "\n\n🔍 Simplebench Extension\n\nThis extension tests AI models across multiple dimensions:\n• Closed-answer contract: 20 deterministic final-line answers (logic, math, spatial, commonsense)\n• Tool Usage: Ability to use available tools effectively\n• Instruction Following: How well the model follows complex JSON instructions\n• Coding Lite: Six isolated, execution-backed coding tasks\n\n📋 Usage Examples:\n  /simplebench                    # Test current model\n  /simplebench <model>           # Test a specific model\n  /simplebench --all             # Test all Ollama models\n  /simplebench <model> --coding-lite # Run only coding tasks\n  /simplebench <model> --test-all # Run baseline, coding, and grounded research\n  /simplebench <model> --research-live # Add live search integration smoke test\n  /simplebench --all --test-all  # Run complete suite for every Ollama model\n  /simplebench <model> --thinking-max # Request max reasoning\n  /simplebench <model> --llama-server # Capture configured /props and /metrics\n  /simplebench <model> --llamagputop # Capture configured llama.cpp stats\n  /simplebench <model> --tag=coldrun # Label the run: benchmark.tag plus artifact filename prefix\n  /simplebench --help            # Show this help\n  /simplebench --clear-cache     # Clear tool support cache\n\nCoding tasks run in disposable directories and never access the user repository.\n",
+  detailedHelp: "\n\n🔍 Simplebench Extension\n\nThis extension tests AI models across multiple dimensions:\n• Closed-answer contract: 20 deterministic final-line answers (logic, math, spatial, commonsense)\n• Tool Usage: Ability to use available tools effectively\n• Instruction Following: How well the model follows complex JSON instructions\n• Coding Lite: Six isolated, execution-backed coding tasks\n\n📋 Usage Examples:\n  /simplebench                    # Test current model\n  /simplebench <model>           # Test a specific model\n  /simplebench --all             # Test all Ollama models\n  /simplebench <model> --3ptest  # Run the default 3ptest baseline explicitly\n  /simplebench <model> --coding-lite # Run only coding tasks\n  /simplebench <model> --test-all # Run baseline, coding, and grounded research\n  /simplebench <model> --research-live # Add live search integration smoke test\n  /simplebench --all --test-all  # Run complete suite for every Ollama model\n  /simplebench <model> --sequence # Run the configured runSequence template\n  /simplebench <model> --thinking-max # Request max reasoning\n  /simplebench <model> --llama-server # Capture configured /props and /metrics\n  /simplebench <model> --llamagputop # Capture configured llama.cpp stats\n  /simplebench <model> --tag=coldrun # Label the run: benchmark.tag plus artifact filename prefix\n  /simplebench --help            # Show this help\n  /simplebench --clear-cache     # Clear tool support cache\n\nCoding tasks run in disposable directories and never access the user repository.\n",
   getArgumentCompletions: async (prefix) => {
     try {
       const models = await getOllamaModels();
@@ -54,6 +72,8 @@ pi.registerCommand("simplebench", {
         "  /simplebench [model] --test-all - Run baseline, coding, and deterministic grounded research\n" +
         "  /simplebench [model] --research-live - Add live-search integration smoke test\n" +
         "  /simplebench [model] --tag=<word> - Label the run (single word; added to benchmark.tag and the artifact name)\n" +
+        "  /simplebench [model] --3ptest - Run the default baseline suite explicitly\n" +
+        "  /simplebench [model] --sequence - Run the templated multi-iteration sequence from opl-simplebench.json (runSequence; per-entry --tag, llamaMetrics and pauseMs supported; the outer --tag is ignored)\n" +
         "  /simplebench --all --test-all - Run complete suite for all Ollama models\n" +
         "  /simplebench --clear-cache - Clear tool support cache\n",
         "info"
@@ -76,6 +96,46 @@ pi.registerCommand("simplebench", {
       return;
     }
 
+    if (parsedArgs.sequence) {
+      let resolved: ResolvedRunSequence;
+      try {
+        resolved = resolveRunSequence(readTestConfig());
+      } catch (e: any) {
+        ctx.ui.notify(e?.message || String(e), "error");
+        return;
+      }
+      const sequenceModel = parsedArgs.model || ctx.model?.id;
+      if (!sequenceModel) {
+        ctx.ui.notify("No model specified and no model currently selected", "error");
+        return;
+      }
+      ctx.ui.notify(`Running ${resolved.runs.length}-iteration sequence (${resolved.runs.map(r => r.entry).join(" | ")})...`, "info");
+      let completed = 0;
+      for (let i = 0; i < resolved.runs.length; i++) {
+        if (i > 0 && resolved.pauseMs > 0) {
+          ctx.ui.notify(`Cooling down ${resolved.pauseMs / 1000}s before iteration ${i + 1}/${resolved.runs.length}...`, "info");
+          await new Promise(resolve => setTimeout(resolve, resolved.pauseMs));
+        }
+        const { entry, options } = resolved.runs[i];
+        const model = options.model || sequenceModel;
+        ctx.ui.notify(`Iteration ${i + 1}/${resolved.runs.length} (${entry}) on ${model}...`, "info");
+        try {
+          const report = await testModel(model, ctx, { ...options, model });
+          completed += 1;
+          pi.sendMessage({
+            customType: "simplebench-report",
+            content: report,
+            display: { type: "content", content: report },
+            details: { model, timestamp: new Date().toISOString() },
+          });
+        } catch (e: any) {
+          ctx.ui.notify(`Iteration ${i + 1} (${entry}) failed: ${e.message}`, "error");
+        }
+      }
+      ctx.ui.notify(`Sequence done: ${completed}/${resolved.runs.length} iterations completed`, "info");
+      return;
+    }
+
     if (parsedArgs.allModels) {
       const providerInfo = detectProvider(ctx);
       if (providerInfo.kind !== "ollama") {

package/extensions/opl-simplebench/types.ts

@@ -1,6 +1,6 @@
 import type { ServerStats } from "./llama-server";
 
-export interface SimplebenchOptions { model?: string; allModels: boolean; writeArtifact: boolean; thinkingMax?: boolean; codingLite?: boolean; testAll?: boolean; researchLive?: boolean; llamaServer?: boolean; llamagputop?: boolean; tag?: string; }
+export interface SimplebenchOptions { model?: string; allModels: boolean; writeArtifact: boolean; thinkingMax?: boolean; codingLite?: boolean; testAll?: boolean; researchLive?: boolean; llamaServer?: boolean; llamagputop?: boolean; tag?: string; threePTest?: boolean; sequence?: boolean; }
 export interface RequestMetrics { requestCount: number; retryCount: number; wallTimeMs: number; timeToAnswerMs: number; timeToFirstTokenMs: number | null; startedAt: string | null; finishedAt: string | null; inputTokens: number | null; outputTokens: number | null; totalTokens: number | null; outputTokensPerSecond: number | null; toolCalls: Array<{ name: string; arguments?: unknown }>; }
 export interface TestRecord { id: string; kind: "reasoning" | "instructions" | "tools" | "coding" | "research-grounded" | "research-live"; category?: string; prompt: string; response: string | null; expectedAnswer?: string | string[]; extractedAnswer?: string; score: string; passed: boolean; error: string | null; metrics: RequestMetrics; coding?: { publicPassed: boolean; hiddenPassed: boolean; verifiedAfterEdit: boolean; unrelatedFiles: string[]; toolCalls: number; turns: number; efficiency?: "STRONG" | "MODERATE" | "WEAK" | "FAIL"; }; research?: { toolCalls: string[]; files: string[]; }; }
 export interface RunArtifact { schemaVersion: 1; benchmark: { name: "opl-simplebench"; suite?: "baseline" | "coding-lite" | "test-all"; tag?: string; model: string; provider: string; providerKind: string; thinking: { requested: "default" | "max"; effective: "provider-default" | "openai-reasoning-effort" | "pi-bedrock-reasoning"; level: "max" | null; modelMetadataSource: "active-context" | "scoped-model" | null; }; startedAt: string; finishedAt: string; wallTimeMs: number; artifactEnabled: boolean; }; tests: TestRecord[]; summary: Record<string, unknown> & { serverStats?: ServerStats }; }

package/extensions/opl-simplebench/util/config.ts

@@ -79,6 +79,15 @@ export interface ModelTestUserConfig {
   researchMaxResults?: number;
   llamaServerUrl?: string;
   llamagputopUrl?: string;
+  runSequence?: RunSequenceConfig;
+}
+
+/** Templated multi-run benchmark sequence used by `/simplebench --sequence`. */
+export interface RunSequenceConfig {
+  enabled?: boolean;
+  llamaMetrics?: boolean;
+  pauseMs?: number;
+  sequence?: string[];
 }
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openlines/opl-pi-sht",
-  "version": "0.1.15",
+  "version": "0.1.16",
   "description": "Pi coding-agent extension kit: token-efficient context, safe modes, fingerprinted /init, model benchmarking, web access, and browser automation.",
   "repository": {
     "type": "git",