@openlines/opl-pi-sht 0.1.15 → 0.1.17

package/README.md

@@ -11,8 +11,8 @@ A portable collection of various Pi coding agent extensions. Repository director
 Install a versioned release from npm or GitHub. Both ship identical content; pick one source per machine, because Pi treats the npm and Git entries as separate packages and installing both loads every extension twice.
 
 ```bash
-pi install npm:@openlines/opl-pi-sht@0.1.15
-pi install git:github.com/linellazatin/opl-pi-sht@v0.1.15
+pi install npm:@openlines/opl-pi-sht@0.1.17
+pi install git:github.com/linellazatin/opl-pi-sht@v0.1.17
 ```
 
 > Omitting the version on the npm source tracks the latest published release; Git refs stay pinned, so move them with `pi install ...@v<new>`.
@@ -34,7 +34,7 @@ npx playwright install chromium
 {
   "packages": [
     {
-      "source": "npm:@openlines/opl-pi-sht@0.1.15",
+      "source": "npm:@openlines/opl-pi-sht@0.1.17",
       "extensions": ["extensions/opl-init/index.ts", "extensions/opl-todo/index.ts"]
     }
   ]
@@ -200,7 +200,7 @@ npx playwright install chromium
 
 A Pi package (npm or Git) still needs the one-time `npx playwright install chromium` command shown above.
 
-`opl-simplebench` writes a full JSON benchmark artifact in Pi's current working directory by default. `--test-all` additionally writes `research.md` and `page.html` beside `result.json` in a result bundle. Copy `configs/opl-simplebench.json.sample` to `~/.pi/agent/configs/opl-simplebench.json` to configure DDGS/SearXNG research and optional llama-server/llamagputop metadata endpoints. Use `/simplebench --no-artifact` or `simplebench({ no_artifact: true })` when responses must not be written to disk. Provider credentials remain outside tracked configuration; configure them through Pi provider settings, environment variables, or Pi authentication.
+`opl-simplebench` writes a full JSON benchmark artifact in Pi's current working directory by default. `--test-all` additionally writes `research.md` and `page.html` beside `result.json` in a result bundle. `/simplebench --sequence[=<name>]` runs a named templated multi-run protocol (for example a model warm-up curve) defined in the `runSequence.sequences` block, with per-iteration tags, optional llama-server/llamagputop metrics, and a configurable pause between iterations; block-level `llamaMetrics`/`pauseMs` are defaults a profile can override. Copy `configs/opl-simplebench.json.sample` to `~/.pi/agent/configs/opl-simplebench.json` to configure DDGS/SearXNG research, optional llama-server/llamagputop metadata endpoints, and the run sequence. Use `/simplebench --no-artifact` or `simplebench({ no_artifact: true })` when responses must not be written to disk. Provider credentials remain outside tracked configuration; configure them through Pi provider settings, environment variables, or Pi authentication.
 
 ## Tests
 
@@ -208,4 +208,4 @@ A Pi package (npm or Git) still needs the one-time `npx playwright install chrom
 npm test
 ```
 
-Run one extension suite with `npm run test:opl-<name>` for `footer`, `init`, `input`, `modes`, `questionnaire`, `todo`, `webaccess`, `simplebench`, or `ctxtrim`. Every helper, functional, and selected-entrypoint smoke check uses Bun's named-test reporter; output includes per-test status, timings, and pass/fail totals. Functional tests cover deterministic helpers where practical; smoke tests bundle entrypoints and parse config. They do not test live TUI behavior, provider credentials, network access, or PDF extraction.
+Run one extension suite with `npm run test:opl-<name>` for `browser`, `footer`, `init`, `input`, `modes`, `questionnaire`, `todo`, `webaccess`, `simplebench`, or `ctxtrim`. Every helper, functional, and selected-entrypoint smoke check uses Bun's named-test reporter; output includes per-test status, timings, and pass/fail totals. Functional tests cover deterministic helpers where practical; smoke tests bundle entrypoints and parse config. They do not test live TUI behavior, provider credentials, network access, or PDF extraction.

package/configs/opl-simplebench.json.sample

@@ -1,8 +1,33 @@
 {
-  "_comment": "Copy to ~/.pi/agent/configs/opl-simplebench.json. Endpoints are optional and only used by their corresponding Simplebench flags.",
+  "_comment": "Copy to ~/.pi/agent/configs/opl-simplebench.json. Endpoints are optional and only used by their corresponding Simplebench flags. runSequence powers /simplebench --sequence[=<name>]: each profile has a single-word name and an iterations list of flag strings; llamaMetrics appends --llama-server --llamagputop to every entry and pauseMs sleeps between iterations. Block-level llamaMetrics/pauseMs are the defaults; a profile may override either. A bare --sequence runs the only defined profile and otherwise lists the names. The legacy flat runSequence.sequence array is still accepted as an anonymous profile.",
   "researchSearchProvider": "ddgs",
   "researchSearchUrl": "http://127.0.0.1:8000",
   "researchMaxResults": 5,
   "llamaServerUrl": "http://127.0.0.1:8080/v1",
-  "llamagputopUrl": "http://127.0.0.1:4321/stats"
+  "llamagputopUrl": "http://127.0.0.1:4321/stats",
+  "runSequence": {
+    "enabled": false,
+    "llamaMetrics": true,
+    "pauseMs": 5000,
+    "sequences": [
+      {
+        "name": "warmup",
+        "iterations": [
+          "--coding-lite --tag=coldest",
+          "--3ptest --tag=colder",
+          "--coding-lite --tag=semiwarm",
+          "--test-all --research-live --tag=warm"
+        ]
+      },
+      {
+        "name": "coding-only",
+        "llamaMetrics": false,
+        "pauseMs": 3000,
+        "iterations": [
+          "--coding-lite --tag=coldest",
+          "--coding-lite --tag=warmer"
+        ]
+      }
+    ]
+  }
 }

package/extensions/opl-simplebench/README.md

@@ -8,12 +8,14 @@ Auditable Pi model benchmark for closed-answer contracts, JSON instruction follo
 /simplebench
 /simplebench <model>
 /simplebench <model> [--no-artifact] [--thinking-max]
+/simplebench <model> --3ptest
 /simplebench <model> --coding-lite
 /simplebench <model> --test-all
 /simplebench <model> --research-live
 /simplebench <model> --llama-server
 /simplebench <model> --llamagputop
 /simplebench <model> --tag=<word>
+/simplebench <model> --sequence[=<name>]
 /simplebench --all [--no-artifact]
 /simplebench --all --test-all
 /simplebench --help
@@ -39,6 +41,8 @@ simplebench({ model: "global.openai.gpt-5.6-terra", test_all: true })
 - Twenty fixed closed-answer contracts, one strict JSON instruction test, and one chained tool-call test.
 - Six opt-in execution-backed coding tasks in disposable directories (`--coding-lite`).
 - `--test-all` runs baseline, coding-lite, and deterministic grounded research; `--research-live` adds the configured live research-artifact integration smoke test. `--all` still selects every Ollama model.
+- `--3ptest` is the explicit form of the default baseline suite (reasoning, instruction following, tool calls). It changes nothing about the run; artifacts already name the baseline suite `3ptest`.
+- `--sequence[=<name>]` runs a named templated multi-iteration profile stored in `opl-simplebench.json` (`runSequence.sequences`): sequential iterations, per-entry flags and `--tag`, `llamaMetrics` and `pauseMs` cooldown with block-level defaults a profile can override. See "Run a templated sequence" below.
 - `--llama-server` (`llama_server: true`) captures configured direct llama-server `/props` and `/metrics` without changing inference routing, so it works with a LiteLLM proxy.
 - `--llamagputop` (`llamagputop: true`) captures configured llama.cpp `/stats` metadata independently of the inference provider; its served model ID is authoritative.
 - `--tag=<word>` (`tag: "<word>"`) labels a run: stored as `benchmark.tag` and prefixed onto the artifact file or bundle name, for example `simplebench-coldrun-coding-lite-<model>-...`. The tag must be a single word (letters, digits, dot, dash, underscore).
@@ -80,6 +84,51 @@ Use this in sensitive environments. The terminal summary still reports category
 
 `--all` is intentionally limited to Ollama discovery. It writes one artifact per model unless `--no-artifact` is set.
 
+### Run a templated sequence
+
+```text
+/simplebench --sequence=<name>
+```
+
+Repeating a warm-up protocol (first-run-after-boot coding check, then baseline, then the full suite) means re-typing the same flags every session. `--sequence` reads a named `runSequence` profile from `~/.pi/agent/configs/opl-simplebench.json` and runs each entry as its own iteration against the current model (or the model passed with `--sequence`):
+
+```json
+{
+  "runSequence": {
+    "enabled": true,
+    "llamaMetrics": true,
+    "pauseMs": 5000,
+    "sequences": [
+      {
+        "name": "warmup",
+        "iterations": [
+          "--coding-lite --tag=coldest",
+          "--3ptest --tag=colder",
+          "--coding-lite --tag=semiwarm",
+          "--test-all --research-live --tag=warm"
+        ]
+      },
+      {
+        "name": "coding-only",
+        "llamaMetrics": false,
+        "pauseMs": 3000,
+        "iterations": ["--coding-lite --tag=coldest", "--coding-lite --tag=warmer"]
+      }
+    ]
+  }
+}
+```
+
+- `enabled` gates the flag: `/simplebench --sequence` errors when it is missing or `false`. A bare `/simplebench` is unaffected.
+- Profile names are single words (letters, digits, dot, dash, underscore) and must be unique. `/simplebench --sequence` without a name runs the only defined profile; with several profiles it errors and lists the available names.
+- Block-level `llamaMetrics` and `pauseMs` are defaults; a profile may override either, so a coding-only curve can skip server stats or use a different cooldown.
+- Each `iterations` entry is the same flag string you would type after `/simplebench`, including its own `--tag`.
+- `llamaMetrics: true` appends `--llama-server --llamagputop` to every entry, so local server stats are captured per iteration without repeating the flags.
+- `pauseMs` sleeps between iterations (never after the last one), including after a failed iteration, so thermal/KV-cache cooldown stays consistent across the warm-up curve.
+- Iterations run sequentially and each writes its normal artifact/bundle; a failed iteration is reported and the sequence continues. A final summary line counts completed iterations.
+- Entries containing `--all` or `--sequence` are rejected before anything runs, as are invalid configurations (no profiles, empty `iterations`, negative `pauseMs`) and invalid per-entry tags. The outer `--tag` is ignored; each entry carries its own.
+- The legacy flat `runSequence.sequence` array still works as an anonymous profile, so existing configs need no change. Selecting it by name is not possible; migrate to `sequences` when you need more than one protocol.
+
 ### Grounded research in `--test-all`
 
 `--test-all` runs a deterministic agent workflow after coding-lite. `web_search` returns three benchmark-local source cards. The model must write every required finding as a Markdown bullet with its inline source ID (`[S1]`, `[S2]`) and list each cited source under `## Sources` as a Markdown link, for example `- [S1](https://research.fixture/S1)`. The verifier checks each exact claim-to-source relationship, source-list URL, tool trace, and the responsive minimalist `page.html` contract. It invokes no LLM judge, external search service, embedding model, or entailment model.
@@ -94,7 +143,7 @@ When coding-lite is enabled, each coding task is recorded as a `kind: "coding"`
 
 ### Configured metadata (`--llama-server`, `--llamagputop`)
 
-Copy `configs/opl-simplebench.json.sample` to `~/.pi/agent/configs/opl-simplebench.json`. It uses camelCase fields: `llamaServerUrl`, `llamagputopUrl`, `researchSearchProvider`, `researchSearchUrl`, and `researchMaxResults`. `--llama-server` captures `GET /props` and `/metrics` from configured `llamaServerUrl`; inference remains on the configured provider route, including LiteLLM. `--llamagputop` captures configured `llamagputopUrl` stats.
+Copy `configs/opl-simplebench.json.sample` to `~/.pi/agent/configs/opl-simplebench.json`. It uses camelCase fields: `llamaServerUrl`, `llamagputopUrl`, `researchSearchProvider`, `researchSearchUrl`, `researchMaxResults`, and `runSequence` (see "Run a templated sequence"). `--llama-server` captures `GET /props` and `/metrics` from configured `llamaServerUrl`; inference remains on the configured provider route, including LiteLLM. `--llamagputop` captures configured `llamagputopUrl` stats.
 
 `serverStats` is omitted entirely for runs without the flag. Its shape:
 

package/extensions/opl-simplebench/benchmark.ts

@@ -138,6 +138,7 @@ function makeOpenAiChatFn(baseUrl: string, apiKey?: string, thinkingMax = false)
 
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), CONFIG.TOOL_TEST_TIMEOUT_MS);
+    const startedAt = new Date().toISOString();
     const start = Date.now();
     try {
       const res = await fetch(`${baseUrl}/chat/completions`, {
@@ -163,6 +164,8 @@ function makeOpenAiChatFn(baseUrl: string, apiKey?: string, thinkingMax = false)
         toolCalls: toolCalls && toolCalls.length > 0 ? toolCalls : undefined,
         elapsedMs,
         raw: parsed,
+        startedAt,
+        finishedAt: new Date().toISOString(),
       };
     } catch (e: any) {
       clearTimeout(timeoutId);
@@ -265,6 +268,7 @@ function makeBedrockChatFn(providerInfo: { region?: string }): ChatFn {
 
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), CONFIG.TOOL_TEST_TIMEOUT_MS);
+    const startedAt = new Date().toISOString();
     const start = Date.now();
     try {
       const res = await fetch(`https://${host}${urlPath}`, {
@@ -284,7 +288,7 @@ function makeBedrockChatFn(providerInfo: { region?: string }): ChatFn {
       const toolCalls = blocks.filter((b: any) => b.toolUse).map((b: any) => ({
         function: { name: b.toolUse.name, arguments: b.toolUse.input ?? {} },
       }));
-      return { content, toolCalls: toolCalls.length > 0 ? toolCalls : undefined, elapsedMs, raw: parsed };
+      return { content, toolCalls: toolCalls.length > 0 ? toolCalls : undefined, elapsedMs, raw: parsed, startedAt, finishedAt: new Date().toISOString() };
     } catch (e: any) {
       clearTimeout(timeoutId);
       if (e.name === "AbortError") throw new Error(`Bedrock API timed out after ${msHuman(CONFIG.TOOL_TEST_TIMEOUT_MS)}`);
@@ -344,6 +348,7 @@ function makeOllamaToolChatFn(): ChatFn {
 
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), CONFIG.TOOL_TEST_TIMEOUT_MS);
+    const startedAt = new Date().toISOString();
     const start = Date.now();
     try {
       const res = await fetch(`${ollamaBase()}/api/chat`, {
@@ -370,6 +375,8 @@ function makeOllamaToolChatFn(): ChatFn {
         toolCalls: toolCalls && toolCalls.length > 0 ? toolCalls : undefined,
         elapsedMs,
         raw: parsed,
+        startedAt,
+        finishedAt: new Date().toISOString(),
       };
     } catch (e: any) {
       clearTimeout(timeoutId);

package/extensions/opl-simplebench/index.ts

@@ -1,7 +1,7 @@
 import type { ExtensionAPI, AgentToolResult } from "@earendil-works/pi-coding-agent";
 import { debugLog } from "./util/debug";
 import { detectProvider } from "./util/providers";
-import { TOOL_SUPPORT_CACHE_PATH } from "./util/config";
+import { readTestConfig, TOOL_SUPPORT_CACHE_PATH, type ModelTestUserConfig, type RunSequenceProfile } from "./util/config";
 import type { SimplebenchOptions } from "./types";
 import { createBenchmark } from "./benchmark";
 
@@ -13,7 +13,50 @@ export function parseCommandArgs(args: string): SimplebenchOptions {
     tag = tagToken.slice("--tag=".length);
     if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(tag)) throw new Error(`--tag must be a single word (letters, digits, dot, dash, underscore): got "${tag}"`);
   }
-  return { model: tokens.find(token => !token.startsWith("--")), allModels: tokens.includes("--all"), writeArtifact: !tokens.includes("--no-artifact"), thinkingMax: tokens.includes("--thinking-max"), codingLite: tokens.includes("--coding-lite"), testAll: tokens.includes("--test-all"), researchLive: tokens.includes("--research-live"), llamaServer: tokens.includes("--llama-server"), llamagputop: tokens.includes("--llamagputop"), ...(tag ? { tag } : {}) };
+  const seqToken = tokens.find(token => token === "--sequence" || token.startsWith("--sequence="));
+  let sequence: boolean | string | undefined;
+  if (seqToken) {
+    if (seqToken === "--sequence") sequence = true;
+    else {
+      sequence = seqToken.slice("--sequence=".length);
+      if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(sequence)) throw new Error(`--sequence= must name a single word (letters, digits, dot, dash, underscore): got "${seqToken.slice("--sequence=".length)}"`);
+    }
+  }
+  return { model: tokens.find(token => !token.startsWith("--")), allModels: tokens.includes("--all"), writeArtifact: !tokens.includes("--no-artifact"), thinkingMax: tokens.includes("--thinking-max"), codingLite: tokens.includes("--coding-lite"), testAll: tokens.includes("--test-all"), researchLive: tokens.includes("--research-live"), llamaServer: tokens.includes("--llama-server"), llamagputop: tokens.includes("--llamagputop"), ...(tag ? { tag } : {}), ...(tokens.includes("--3ptest") ? { threePTest: true } : {}), ...(sequence ? { sequence } : {}) };
+}
+
+export interface ResolvedRunSequence { profile: string; runs: Array<{ entry: string; options: SimplebenchOptions }>; pauseMs: number; llamaMetrics: boolean }
+
+const PROFILE_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+
+/** Expand one configured runSequence profile into per-iteration benchmark options. Throws on disabled/invalid configuration. */
+export function resolveRunSequence(userConfig: ModelTestUserConfig, name?: string): ResolvedRunSequence {
+  const rs = userConfig.runSequence;
+  if (!rs?.enabled) throw new Error("runSequence is not enabled in opl-simplebench.json");
+  const profiles: RunSequenceProfile[] = [];
+  if (Array.isArray(rs.sequence) && rs.sequence.length > 0) profiles.push({ name: "", iterations: rs.sequence });
+  for (const profile of rs.sequences ?? []) {
+    if (typeof profile?.name !== "string" || !PROFILE_NAME_RE.test(profile.name)) throw new Error(`runSequence profile name must be a single word (letters, digits, dot, dash, underscore): got ${JSON.stringify(profile?.name)}`);
+    if (!Array.isArray(profile.iterations) || profile.iterations.length === 0) throw new Error(`runSequence profile "${profile.name}" needs a non-empty iterations array`);
+    if (profiles.some(known => known.name === profile.name)) throw new Error(`duplicate runSequence profile name: "${profile.name}"`);
+    profiles.push(profile);
+  }
+  const label = (p: RunSequenceProfile) => p.name || "(legacy)";
+  if (profiles.length === 0) throw new Error("runSequence needs a non-empty sequence array or sequences list");
+  let chosen: RunSequenceProfile | undefined;
+  if (name) chosen = profiles.find(p => p.name === name);
+  else if (profiles.length === 1) chosen = profiles[0];
+  if (!chosen) throw new Error(name ? `unknown runSequence profile "${name}"; available: ${profiles.map(label).join(", ")}` : `--sequence needs a profile name; available: ${profiles.map(label).join(", ")}`);
+  const pauseMs = chosen.pauseMs ?? rs.pauseMs ?? 0;
+  if (typeof pauseMs !== "number" || !Number.isFinite(pauseMs) || pauseMs < 0) throw new Error(`runSequence.pauseMs must be a non-negative number: got ${JSON.stringify(chosen.pauseMs ?? rs.pauseMs)}`);
+  const llamaMetrics = (chosen.llamaMetrics ?? rs.llamaMetrics) === true;
+  const runs = chosen.iterations.map((entry: string) => {
+    const tokens = String(entry).trim().split(/\s+/);
+    if (tokens.includes("--all")) throw new Error(`runSequence entry cannot contain --all: "${entry}"`);
+    if (tokens.some(token => token === "--sequence" || token.startsWith("--sequence="))) throw new Error(`runSequence entry cannot contain --sequence: "${entry}"`);
+    return { entry: String(entry), options: parseCommandArgs(`${entry}${llamaMetrics ? " --llama-server --llamagputop" : ""}`) };
+  });
+  return { profile: chosen.name, runs, pauseMs, llamaMetrics };
 }
 
 export default function (pi: ExtensionAPI) {
@@ -22,7 +65,7 @@ export default function (pi: ExtensionAPI) {
 
 pi.registerCommand("simplebench", {
   description: "Benchmark a model with auditable closed-answer, instruction, and tool-use tests.",
-  detailedHelp: "\n\n🔍 Simplebench Extension\n\nThis extension tests AI models across multiple dimensions:\n• Closed-answer contract: 20 deterministic final-line answers (logic, math, spatial, commonsense)\n• Tool Usage: Ability to use available tools effectively\n• Instruction Following: How well the model follows complex JSON instructions\n• Coding Lite: Six isolated, execution-backed coding tasks\n\n📋 Usage Examples:\n  /simplebench                    # Test current model\n  /simplebench <model>           # Test a specific model\n  /simplebench --all             # Test all Ollama models\n  /simplebench <model> --coding-lite # Run only coding tasks\n  /simplebench <model> --test-all # Run baseline, coding, and grounded research\n  /simplebench <model> --research-live # Add live search integration smoke test\n  /simplebench --all --test-all  # Run complete suite for every Ollama model\n  /simplebench <model> --thinking-max # Request max reasoning\n  /simplebench <model> --llama-server # Capture configured /props and /metrics\n  /simplebench <model> --llamagputop # Capture configured llama.cpp stats\n  /simplebench <model> --tag=coldrun # Label the run: benchmark.tag plus artifact filename prefix\n  /simplebench --help            # Show this help\n  /simplebench --clear-cache     # Clear tool support cache\n\nCoding tasks run in disposable directories and never access the user repository.\n",
+  detailedHelp: "\n\n🔍 Simplebench Extension\n\nThis extension tests AI models across multiple dimensions:\n• Closed-answer contract: 20 deterministic final-line answers (logic, math, spatial, commonsense)\n• Tool Usage: Ability to use available tools effectively\n• Instruction Following: How well the model follows complex JSON instructions\n• Coding Lite: Six isolated, execution-backed coding tasks\n\n📋 Usage Examples:\n  /simplebench                    # Test current model\n  /simplebench <model>           # Test a specific model\n  /simplebench --all             # Test all Ollama models\n  /simplebench <model> --3ptest  # Run the default 3ptest baseline explicitly\n  /simplebench <model> --coding-lite # Run only coding tasks\n  /simplebench <model> --test-all # Run baseline, coding, and grounded research\n  /simplebench <model> --research-live # Add live search integration smoke test\n  /simplebench --all --test-all  # Run complete suite for every Ollama model\n  /simplebench <model> --sequence[=<name>] # Run a configured runSequence profile\n  /simplebench <model> --thinking-max # Request max reasoning\n  /simplebench <model> --llama-server # Capture configured /props and /metrics\n  /simplebench <model> --llamagputop # Capture configured llama.cpp stats\n  /simplebench <model> --tag=coldrun # Label the run: benchmark.tag plus artifact filename prefix\n  /simplebench --help            # Show this help\n  /simplebench --clear-cache     # Clear tool support cache\n\nCoding tasks run in disposable directories and never access the user repository.\n",
   getArgumentCompletions: async (prefix) => {
     try {
       const models = await getOllamaModels();
@@ -54,6 +97,8 @@ pi.registerCommand("simplebench", {
         "  /simplebench [model] --test-all - Run baseline, coding, and deterministic grounded research\n" +
         "  /simplebench [model] --research-live - Add live-search integration smoke test\n" +
         "  /simplebench [model] --tag=<word> - Label the run (single word; added to benchmark.tag and the artifact name)\n" +
+        "  /simplebench [model] --3ptest - Run the default baseline suite explicitly\n" +
+        "  /simplebench [model] --sequence[=<name>] - Run a templated multi-iteration sequence profile from opl-simplebench.json (runSequence.sequences; bare --sequence needs exactly one profile; per-entry --tag, llamaMetrics and pauseMs supported, profile values override the block ones; the outer --tag is ignored)\n" +
         "  /simplebench --all --test-all - Run complete suite for all Ollama models\n" +
         "  /simplebench --clear-cache - Clear tool support cache\n",
         "info"
@@ -76,6 +121,47 @@ pi.registerCommand("simplebench", {
       return;
     }
 
+    if (parsedArgs.sequence) {
+      const requested = typeof parsedArgs.sequence === "string" ? parsedArgs.sequence : undefined;
+      let resolved: ResolvedRunSequence;
+      try {
+        resolved = resolveRunSequence(readTestConfig(), requested);
+      } catch (e: any) {
+        ctx.ui.notify(e?.message || String(e), "error");
+        return;
+      }
+      const sequenceModel = parsedArgs.model || ctx.model?.id;
+      if (!sequenceModel) {
+        ctx.ui.notify("No model specified and no model currently selected", "error");
+        return;
+      }
+      ctx.ui.notify(`Running ${resolved.runs.length}-iteration sequence${resolved.profile ? ` "${resolved.profile}"` : ""} (llamaMetrics ${resolved.llamaMetrics ? "on" : "off"}, pause ${resolved.pauseMs / 1000}s): ${resolved.runs.map(r => r.entry).join(" | ")}...`, "info");
+      let completed = 0;
+      for (let i = 0; i < resolved.runs.length; i++) {
+        if (i > 0 && resolved.pauseMs > 0) {
+          ctx.ui.notify(`Cooling down ${resolved.pauseMs / 1000}s before iteration ${i + 1}/${resolved.runs.length}...`, "info");
+          await new Promise(resolve => setTimeout(resolve, resolved.pauseMs));
+        }
+        const { entry, options } = resolved.runs[i];
+        const model = options.model || sequenceModel;
+        ctx.ui.notify(`Iteration ${i + 1}/${resolved.runs.length} (${entry}) on ${model}...`, "info");
+        try {
+          const report = await testModel(model, ctx, { ...options, model });
+          completed += 1;
+          pi.sendMessage({
+            customType: "simplebench-report",
+            content: report,
+            display: { type: "content", content: report },
+            details: { model, timestamp: new Date().toISOString() },
+          });
+        } catch (e: any) {
+          ctx.ui.notify(`Iteration ${i + 1} (${entry}) failed: ${e.message}`, "error");
+        }
+      }
+      ctx.ui.notify(`Sequence done: ${completed}/${resolved.runs.length} iterations completed`, "info");
+      return;
+    }
+
     if (parsedArgs.allModels) {
       const providerInfo = detectProvider(ctx);
       if (providerInfo.kind !== "ollama") {

package/extensions/opl-simplebench/types.ts

@@ -1,6 +1,6 @@
 import type { ServerStats } from "./llama-server";
 
-export interface SimplebenchOptions { model?: string; allModels: boolean; writeArtifact: boolean; thinkingMax?: boolean; codingLite?: boolean; testAll?: boolean; researchLive?: boolean; llamaServer?: boolean; llamagputop?: boolean; tag?: string; }
+export interface SimplebenchOptions { model?: string; allModels: boolean; writeArtifact: boolean; thinkingMax?: boolean; codingLite?: boolean; testAll?: boolean; researchLive?: boolean; llamaServer?: boolean; llamagputop?: boolean; tag?: string; threePTest?: boolean; sequence?: boolean | string; }
 export interface RequestMetrics { requestCount: number; retryCount: number; wallTimeMs: number; timeToAnswerMs: number; timeToFirstTokenMs: number | null; startedAt: string | null; finishedAt: string | null; inputTokens: number | null; outputTokens: number | null; totalTokens: number | null; outputTokensPerSecond: number | null; toolCalls: Array<{ name: string; arguments?: unknown }>; }
 export interface TestRecord { id: string; kind: "reasoning" | "instructions" | "tools" | "coding" | "research-grounded" | "research-live"; category?: string; prompt: string; response: string | null; expectedAnswer?: string | string[]; extractedAnswer?: string; score: string; passed: boolean; error: string | null; metrics: RequestMetrics; coding?: { publicPassed: boolean; hiddenPassed: boolean; verifiedAfterEdit: boolean; unrelatedFiles: string[]; toolCalls: number; turns: number; efficiency?: "STRONG" | "MODERATE" | "WEAK" | "FAIL"; }; research?: { toolCalls: string[]; files: string[]; }; }
 export interface RunArtifact { schemaVersion: 1; benchmark: { name: "opl-simplebench"; suite?: "baseline" | "coding-lite" | "test-all"; tag?: string; model: string; provider: string; providerKind: string; thinking: { requested: "default" | "max"; effective: "provider-default" | "openai-reasoning-effort" | "pi-bedrock-reasoning"; level: "max" | null; modelMetadataSource: "active-context" | "scoped-model" | null; }; startedAt: string; finishedAt: string; wallTimeMs: number; artifactEnabled: boolean; }; tests: TestRecord[]; summary: Record<string, unknown> & { serverStats?: ServerStats }; }

package/extensions/opl-simplebench/util/config.ts

@@ -79,6 +79,26 @@ export interface ModelTestUserConfig {
   researchMaxResults?: number;
   llamaServerUrl?: string;
   llamagputopUrl?: string;
+  runSequence?: RunSequenceConfig;
+}
+
+/** Templated multi-run benchmark sequence used by `/simplebench --sequence`. */
+export interface RunSequenceConfig {
+  enabled?: boolean;
+  llamaMetrics?: boolean;
+  pauseMs?: number;
+  /** Legacy anonymous sequence; still resolved when no `sequences` profile matches. */
+  sequence?: string[];
+  /** Named sequences selectable with `/simplebench --sequence=<name>`. */
+  sequences?: RunSequenceProfile[];
+}
+
+/** One named sequence profile. `llamaMetrics` and `pauseMs` override the block-level values. */
+export interface RunSequenceProfile {
+  name: string;
+  iterations: string[];
+  llamaMetrics?: boolean;
+  pauseMs?: number;
 }
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openlines/opl-pi-sht",
-  "version": "0.1.15",
+  "version": "0.1.17",
   "description": "Pi coding-agent extension kit: token-efficient context, safe modes, fingerprinted /init, model benchmarking, web access, and browser automation.",
   "repository": {
     "type": "git",